Model save

README.md

@@ -18,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.1132
-- Rewards/chosen: -0.3948
-- Rewards/rejected: -0.9598
-- Rewards/accuracies: 0.7543
-- Rewards/margins: 0.5650
-- Logps/rejected: -342.4998
-- Logps/chosen: -324.5692
-- Logits/rejected: 1.4353
-- Logits/chosen: 0.4067
+- Loss: 0.1126
+- Rewards/chosen: -0.4018
+- Rewards/rejected: -0.9642
+- Rewards/accuracies: 0.75
+- Rewards/margins: 0.5623
+- Logps/rejected: -342.9399
+- Logps/chosen: -325.2745
+- Logits/rejected: 0.1904
+- Logits/chosen: -0.6159
 
 ## Model description
 
@@ -63,14 +63,14 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1732        | 0.1147 | 50   | 0.1641          | 0.0166         | -0.1092          | 0.7069             | 0.1259          | -257.4446      | -283.4267    | -2.4943         | -2.5737       |
-| 0.1412        | 0.2294 | 100  | 0.1347          | -0.4342        | -0.7962          | 0.7284             | 0.3620          | -326.1406      | -328.5106    | -0.1538         | -0.4184       |
-| 0.1307        | 0.3440 | 150  | 0.1261          | -0.3553        | -0.8583          | 0.7284             | 0.5030          | -332.3533      | -320.6210    | 0.7144          | 0.0181        |
-| 0.1238        | 0.4587 | 200  | 0.1199          | -0.4108        | -0.9476          | 0.7328             | 0.5368          | -341.2862      | -326.1717    | 1.2989          | 0.2969        |
-| 0.1185        | 0.5734 | 250  | 0.1166          | -0.3086        | -0.8924          | 0.7543             | 0.5838          | -335.7633      | -315.9550    | 0.8516          | -0.1745       |
-| 0.1228        | 0.6881 | 300  | 0.1155          | -0.3695        | -0.9267          | 0.7457             | 0.5571          | -339.1875      | -322.0434    | 0.8574          | -0.1316       |
-| 0.1213        | 0.8028 | 350  | 0.1136          | -0.4396        | -1.0157          | 0.7629             | 0.5762          | -348.0973      | -329.0486    | 1.5740          | 0.5152        |
-| 0.12          | 0.9174 | 400  | 0.1132          | -0.3948        | -0.9598          | 0.7543             | 0.5650          | -342.4998      | -324.5692    | 1.4353          | 0.4067        |
+| 0.1732        | 0.1147 | 50   | 0.1640          | 0.0140         | -0.1117          | 0.7069             | 0.1256          | -257.6913      | -283.6945    | -2.4927         | -2.5723       |
+| 0.1412        | 0.2294 | 100  | 0.1392          | -0.5601        | -0.9388          | 0.7198             | 0.3787          | -340.4027      | -341.0970    | -0.8338         | -1.1029       |
+| 0.1327        | 0.3440 | 150  | 0.1257          | -0.3660        | -0.8686          | 0.7241             | 0.5025          | -333.3776      | -321.6920    | -0.4213         | -0.9346       |
+| 0.1242        | 0.4587 | 200  | 0.1193          | -0.3424        | -0.8506          | 0.7457             | 0.5082          | -331.5807      | -319.3252    | -0.1498         | -0.9084       |
+| 0.118         | 0.5734 | 250  | 0.1167          | -0.3734        | -0.9685          | 0.7543             | 0.5950          | -343.3683      | -322.4342    | -0.0251         | -0.8147       |
+| 0.1236        | 0.6881 | 300  | 0.1145          | -0.4780        | -1.0449          | 0.7629             | 0.5669          | -351.0162      | -332.8936    | 0.2891          | -0.5285       |
+| 0.122         | 0.8028 | 350  | 0.1127          | -0.4577        | -1.0376          | 0.7672             | 0.5799          | -350.2830      | -330.8570    | 0.3397          | -0.4806       |
+| 0.12          | 0.9174 | 400  | 0.1126          | -0.4018        | -0.9642          | 0.75               | 0.5623          | -342.9399      | -325.2745    | 0.1904          | -0.6159       |
 
 
 ### Framework versions

all_results.json

@@ -1,22 +1,22 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -1.684370994567871,
-    "eval_logits/rejected": -1.4736462831497192,
-    "eval_logps/chosen": -297.6668701171875,
-    "eval_logps/rejected": -289.2203674316406,
-    "eval_loss": 0.032715052366256714,
+    "eval_logits/chosen": 0.41779446601867676,
+    "eval_logits/rejected": 1.454390525817871,
+    "eval_logps/chosen": -324.46234130859375,
+    "eval_logps/rejected": -342.5584716796875,
+    "eval_loss": 0.11311393231153488,
     "eval_rewards/accuracies": 0.7543103694915771,
-    "eval_rewards/chosen": -0.1257677674293518,
-    "eval_rewards/margins": 0.30121153593063354,
-    "eval_rewards/rejected": -0.42697927355766296,
-    "eval_runtime": 94.8006,
+    "eval_rewards/chosen": -0.393722802400589,
+    "eval_rewards/margins": 0.566637396812439,
+    "eval_rewards/rejected": -0.9603601098060608,
+    "eval_runtime": 93.576,
     "eval_samples": 1818,
-    "eval_samples_per_second": 19.177,
-    "eval_steps_per_second": 0.306,
+    "eval_samples_per_second": 19.428,
+    "eval_steps_per_second": 0.31,
     "total_flos": 0.0,
-    "train_loss": 0.13218348616853767,
-    "train_runtime": 11343.5766,
+    "train_loss": 0.13197535372108493,
+    "train_runtime": 11523.0815,
     "train_samples": 55758,
-    "train_samples_per_second": 4.915,
+    "train_samples_per_second": 4.839,
     "train_steps_per_second": 0.038
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -1.684370994567871,
-    "eval_logits/rejected": -1.4736462831497192,
-    "eval_logps/chosen": -297.6668701171875,
-    "eval_logps/rejected": -289.2203674316406,
-    "eval_loss": 0.032715052366256714,
+    "eval_logits/chosen": 0.41779446601867676,
+    "eval_logits/rejected": 1.454390525817871,
+    "eval_logps/chosen": -324.46234130859375,
+    "eval_logps/rejected": -342.5584716796875,
+    "eval_loss": 0.11311393231153488,
     "eval_rewards/accuracies": 0.7543103694915771,
-    "eval_rewards/chosen": -0.1257677674293518,
-    "eval_rewards/margins": 0.30121153593063354,
-    "eval_rewards/rejected": -0.42697927355766296,
-    "eval_runtime": 94.8006,
+    "eval_rewards/chosen": -0.393722802400589,
+    "eval_rewards/margins": 0.566637396812439,
+    "eval_rewards/rejected": -0.9603601098060608,
+    "eval_runtime": 93.576,
     "eval_samples": 1818,
-    "eval_samples_per_second": 19.177,
-    "eval_steps_per_second": 0.306
+    "eval_samples_per_second": 19.428,
+    "eval_steps_per_second": 0.31
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5905b6fface4e45339623c4d375f9edc78e435e73d598309e1ed43118ed92014
+oid sha256:1e0e9f49e5125e802deca218e3a20f75ae9597d6c0bfe6256478d7a39e082ac0
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b04ddbeebeda55ae3848556f3a71f59213ac25a02039521f54874d8cdbff36f1
+oid sha256:7e9cabac2e92bfb43293f329c88741c80f6d4c2811430c129984ce94af6a917d
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b01821c5089f266496aec2a04ad6bae11924a4608f9a7968429e02dcfedff8ea
+oid sha256:50e47b05e3ffc16c276de51ad276bb767d2f2baf1107bb39525ad6872def7e26
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.13218348616853767,
-    "train_runtime": 11343.5766,
+    "train_loss": 0.13197535372108493,
+    "train_runtime": 11523.0815,
     "train_samples": 55758,
-    "train_samples_per_second": 4.915,
+    "train_samples_per_second": 4.839,
     "train_steps_per_second": 0.038
 }

trainer_state.json

@@ -10,784 +10,784 @@
   "log_history": [
     {
       "epoch": 0.022935779816513763,
-      "grad_norm": 5.597671583305064,
+      "grad_norm": 5.604829014017037,
       "learning_rate": 1.1363636363636363e-07,
-      "logits/chosen": -2.6193342208862305,
-      "logits/rejected": -2.5525386333465576,
-      "logps/chosen": -265.3854064941406,
-      "logps/rejected": -236.1589813232422,
-      "loss": 0.1941,
-      "rewards/accuracies": 0.41874998807907104,
-      "rewards/chosen": 0.0006044252077117562,
-      "rewards/margins": 0.0008946189773268998,
-      "rewards/rejected": -0.000290193798718974,
+      "logits/chosen": -2.619293689727783,
+      "logits/rejected": -2.5524563789367676,
+      "logps/chosen": -265.4638977050781,
+      "logps/rejected": -236.136962890625,
+      "loss": 0.1942,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.00018078314315062016,
+      "rewards/margins": -0.00011108322360087186,
+      "rewards/rejected": -6.969989772187546e-05,
       "step": 10
     },
     {
       "epoch": 0.045871559633027525,
-      "grad_norm": 5.252146327563864,
+      "grad_norm": 5.250583616242791,
       "learning_rate": 2.2727272727272726e-07,
-      "logits/chosen": -2.657933473587036,
-      "logits/rejected": -2.576028347015381,
-      "logps/chosen": -298.8043518066406,
-      "logps/rejected": -274.30718994140625,
-      "loss": 0.1958,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.00019080772472079843,
-      "rewards/margins": 0.0017816700274124742,
-      "rewards/rejected": -0.001972477650269866,
+      "logits/chosen": -2.6577353477478027,
+      "logits/rejected": -2.575864315032959,
+      "logps/chosen": -298.8517150878906,
+      "logps/rejected": -274.293212890625,
+      "loss": 0.1959,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0006643417873419821,
+      "rewards/margins": 0.0011681303149089217,
+      "rewards/rejected": -0.0018324721604585648,
       "step": 20
     },
     {
       "epoch": 0.06880733944954129,
-      "grad_norm": 4.974318928321663,
+      "grad_norm": 4.980077918182304,
       "learning_rate": 3.4090909090909085e-07,
-      "logits/chosen": -2.6759729385375977,
-      "logits/rejected": -2.6023097038269043,
-      "logps/chosen": -290.3868713378906,
-      "logps/rejected": -234.3571319580078,
+      "logits/chosen": -2.675644636154175,
+      "logits/rejected": -2.6021487712860107,
+      "logps/chosen": -290.4170227050781,
+      "logps/rejected": -234.3779296875,
       "loss": 0.1923,
       "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.009458948858082294,
-      "rewards/margins": 0.013211230747401714,
-      "rewards/rejected": -0.0037522814236581326,
+      "rewards/chosen": 0.009157607331871986,
+      "rewards/margins": 0.013118197210133076,
+      "rewards/rejected": -0.003960589878261089,
       "step": 30
     },
     {
       "epoch": 0.09174311926605505,
-      "grad_norm": 5.209557025843711,
+      "grad_norm": 5.215480488397355,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": -2.659982204437256,
-      "logits/rejected": -2.6102194786071777,
-      "logps/chosen": -280.9975891113281,
-      "logps/rejected": -267.7278747558594,
-      "loss": 0.1836,
+      "logits/chosen": -2.660599946975708,
+      "logits/rejected": -2.6108055114746094,
+      "logps/chosen": -281.02667236328125,
+      "logps/rejected": -267.7845764160156,
+      "loss": 0.1835,
       "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.04171619936823845,
-      "rewards/margins": 0.041064582765102386,
-      "rewards/rejected": 0.0006516153225675225,
+      "rewards/chosen": 0.04142485558986664,
+      "rewards/margins": 0.04134037345647812,
+      "rewards/rejected": 8.447784784948453e-05,
       "step": 40
     },
     {
       "epoch": 0.11467889908256881,
-      "grad_norm": 5.582751836908906,
+      "grad_norm": 5.6967811731593905,
       "learning_rate": 4.997110275491701e-07,
-      "logits/chosen": -2.6198437213897705,
-      "logits/rejected": -2.6123862266540527,
-      "logps/chosen": -293.4254455566406,
-      "logps/rejected": -303.90386962890625,
+      "logits/chosen": -2.619861602783203,
+      "logits/rejected": -2.6124091148376465,
+      "logps/chosen": -293.50762939453125,
+      "logps/rejected": -303.9714660644531,
       "loss": 0.1732,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.028257910162210464,
-      "rewards/margins": 0.0754549652338028,
-      "rewards/rejected": -0.047197047621011734,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.027435380965471268,
+      "rewards/margins": 0.0753081664443016,
+      "rewards/rejected": -0.04787278175354004,
       "step": 50
     },
     {
       "epoch": 0.11467889908256881,
-      "eval_logits/chosen": -2.5737149715423584,
-      "eval_logits/rejected": -2.494300365447998,
-      "eval_logps/chosen": -283.4266662597656,
-      "eval_logps/rejected": -257.4446105957031,
-      "eval_loss": 0.16405636072158813,
+      "eval_logits/chosen": -2.5723471641540527,
+      "eval_logits/rejected": -2.49267315864563,
+      "eval_logps/chosen": -283.6944580078125,
+      "eval_logps/rejected": -257.6913146972656,
+      "eval_loss": 0.1639617681503296,
       "eval_rewards/accuracies": 0.7068965435028076,
-      "eval_rewards/chosen": 0.016634328290820122,
-      "eval_rewards/margins": 0.12585583329200745,
-      "eval_rewards/rejected": -0.10922150313854218,
-      "eval_runtime": 93.7026,
-      "eval_samples_per_second": 19.402,
-      "eval_steps_per_second": 0.309,
+      "eval_rewards/chosen": 0.013956375420093536,
+      "eval_rewards/margins": 0.12564486265182495,
+      "eval_rewards/rejected": -0.11168847978115082,
+      "eval_runtime": 95.9923,
+      "eval_samples_per_second": 18.939,
+      "eval_steps_per_second": 0.302,
       "step": 50
     },
     {
       "epoch": 0.13761467889908258,
-      "grad_norm": 7.241200335566244,
+      "grad_norm": 7.248589740263705,
       "learning_rate": 4.979475034558115e-07,
-      "logits/chosen": -2.562969207763672,
-      "logits/rejected": -2.5053610801696777,
-      "logps/chosen": -293.1821594238281,
-      "logps/rejected": -272.9414367675781,
-      "loss": 0.1593,
+      "logits/chosen": -2.5621228218078613,
+      "logits/rejected": -2.503438949584961,
+      "logps/chosen": -293.131103515625,
+      "logps/rejected": -272.94793701171875,
+      "loss": 0.1592,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.03697499260306358,
-      "rewards/margins": 0.11687393486499786,
-      "rewards/rejected": -0.15384894609451294,
+      "rewards/chosen": -0.03646404668688774,
+      "rewards/margins": 0.11744984239339828,
+      "rewards/rejected": -0.15391388535499573,
       "step": 60
     },
     {
       "epoch": 0.16055045871559634,
-      "grad_norm": 7.058665086800562,
+      "grad_norm": 7.653772817525038,
       "learning_rate": 4.945923025551788e-07,
-      "logits/chosen": -2.4592204093933105,
-      "logits/rejected": -2.4099252223968506,
-      "logps/chosen": -327.544189453125,
-      "logps/rejected": -284.48760986328125,
-      "loss": 0.1564,
+      "logits/chosen": -2.4572176933288574,
+      "logits/rejected": -2.407254219055176,
+      "logps/chosen": -327.6614685058594,
+      "logps/rejected": -284.7535705566406,
+      "loss": 0.1557,
       "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.00026793667348101735,
-      "rewards/margins": 0.23106786608695984,
-      "rewards/rejected": -0.23079994320869446,
+      "rewards/chosen": -0.0009046800551004708,
+      "rewards/margins": 0.23255476355552673,
+      "rewards/rejected": -0.23345942795276642,
       "step": 70
     },
     {
       "epoch": 0.1834862385321101,
-      "grad_norm": 8.05919017424508,
+      "grad_norm": 8.127224192262021,
       "learning_rate": 4.896669632591651e-07,
-      "logits/chosen": -2.3360755443573,
-      "logits/rejected": -2.2115492820739746,
-      "logps/chosen": -298.69482421875,
-      "logps/rejected": -292.111328125,
-      "loss": 0.1478,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.128449946641922,
-      "rewards/margins": 0.26610296964645386,
-      "rewards/rejected": -0.39455294609069824,
+      "logits/chosen": -2.3246560096740723,
+      "logits/rejected": -2.2035281658172607,
+      "logps/chosen": -297.11651611328125,
+      "logps/rejected": -290.16302490234375,
+      "loss": 0.1475,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.11266676336526871,
+      "rewards/margins": 0.2624030113220215,
+      "rewards/rejected": -0.3750697076320648,
       "step": 80
     },
     {
       "epoch": 0.20642201834862386,
-      "grad_norm": 8.402102216426163,
+      "grad_norm": 8.777001707426516,
       "learning_rate": 4.832031033425662e-07,
-      "logits/chosen": -1.9170547723770142,
-      "logits/rejected": -1.8550583124160767,
-      "logps/chosen": -299.20281982421875,
-      "logps/rejected": -289.49163818359375,
-      "loss": 0.1441,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.1160941943526268,
-      "rewards/margins": 0.3108092248439789,
-      "rewards/rejected": -0.4269034266471863,
+      "logits/chosen": -1.9738223552703857,
+      "logits/rejected": -1.9115241765975952,
+      "logps/chosen": -299.46820068359375,
+      "logps/rejected": -291.00994873046875,
+      "loss": 0.1449,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.11874761432409286,
+      "rewards/margins": 0.3233392536640167,
+      "rewards/rejected": -0.4420868456363678,
       "step": 90
     },
     {
       "epoch": 0.22935779816513763,
-      "grad_norm": 8.625504630985954,
+      "grad_norm": 9.678897311244762,
       "learning_rate": 4.752422169756047e-07,
-      "logits/chosen": -0.48900121450424194,
-      "logits/rejected": -0.44961875677108765,
-      "logps/chosen": -311.1794738769531,
-      "logps/rejected": -339.92254638671875,
+      "logits/chosen": -1.0515625476837158,
+      "logits/rejected": -1.0063254833221436,
+      "logps/chosen": -311.99945068359375,
+      "logps/rejected": -342.3149719238281,
       "loss": 0.1412,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.3875480592250824,
-      "rewards/margins": 0.2923927903175354,
-      "rewards/rejected": -0.6799408197402954,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.39574748277664185,
+      "rewards/margins": 0.3081180453300476,
+      "rewards/rejected": -0.7038655281066895,
       "step": 100
     },
     {
       "epoch": 0.22935779816513763,
-      "eval_logits/chosen": -0.4184306561946869,
-      "eval_logits/rejected": -0.1537734419107437,
-      "eval_logps/chosen": -328.5106201171875,
-      "eval_logps/rejected": -326.1405944824219,
-      "eval_loss": 0.13468901813030243,
-      "eval_rewards/accuracies": 0.7284482717514038,
-      "eval_rewards/chosen": -0.4342052936553955,
-      "eval_rewards/margins": 0.3619759678840637,
-      "eval_rewards/rejected": -0.796181321144104,
-      "eval_runtime": 92.0893,
-      "eval_samples_per_second": 19.742,
-      "eval_steps_per_second": 0.315,
+      "eval_logits/chosen": -1.1028852462768555,
+      "eval_logits/rejected": -0.8337642550468445,
+      "eval_logps/chosen": -341.0970458984375,
+      "eval_logps/rejected": -340.4026794433594,
+      "eval_loss": 0.13921599090099335,
+      "eval_rewards/accuracies": 0.7198275923728943,
+      "eval_rewards/chosen": -0.5600696802139282,
+      "eval_rewards/margins": 0.3787323534488678,
+      "eval_rewards/rejected": -0.9388020038604736,
+      "eval_runtime": 95.7518,
+      "eval_samples_per_second": 18.987,
+      "eval_steps_per_second": 0.303,
       "step": 100
     },
     {
       "epoch": 0.25229357798165136,
-      "grad_norm": 10.103043851045701,
+      "grad_norm": 8.075119669591446,
       "learning_rate": 4.658354083558188e-07,
-      "logits/chosen": -0.40221118927001953,
-      "logits/rejected": -0.13967064023017883,
-      "logps/chosen": -312.624267578125,
-      "logps/rejected": -311.3497009277344,
-      "loss": 0.1354,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.4743613302707672,
-      "rewards/margins": 0.3018040060997009,
-      "rewards/rejected": -0.776165246963501,
+      "logits/chosen": -1.309912085533142,
+      "logits/rejected": -1.0928936004638672,
+      "logps/chosen": -323.1109313964844,
+      "logps/rejected": -320.5851135253906,
+      "loss": 0.1371,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5792280435562134,
+      "rewards/margins": 0.2892912030220032,
+      "rewards/rejected": -0.868519127368927,
       "step": 110
     },
     {
       "epoch": 0.27522935779816515,
-      "grad_norm": 10.671011000324334,
+      "grad_norm": 11.527192184915142,
       "learning_rate": 4.550430636492389e-07,
-      "logits/chosen": -0.04719700291752815,
-      "logits/rejected": 0.09439365565776825,
-      "logps/chosen": -324.4268798828125,
-      "logps/rejected": -335.16424560546875,
-      "loss": 0.1353,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.4860754609107971,
-      "rewards/margins": 0.3509276807308197,
-      "rewards/rejected": -0.8370031118392944,
+      "logits/chosen": -1.2554824352264404,
+      "logits/rejected": -1.2111155986785889,
+      "logps/chosen": -315.65582275390625,
+      "logps/rejected": -323.9941101074219,
+      "loss": 0.1347,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.39836472272872925,
+      "rewards/margins": 0.3269372582435608,
+      "rewards/rejected": -0.7253020405769348,
       "step": 120
     },
     {
       "epoch": 0.2981651376146789,
-      "grad_norm": 11.348522180816168,
+      "grad_norm": 10.253713539855786,
       "learning_rate": 4.429344633468004e-07,
-      "logits/chosen": -0.058751799166202545,
-      "logits/rejected": 0.29970529675483704,
-      "logps/chosen": -296.40399169921875,
-      "logps/rejected": -316.8427429199219,
-      "loss": 0.1373,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.4022388458251953,
-      "rewards/margins": 0.3400627374649048,
-      "rewards/rejected": -0.7423015832901001,
+      "logits/chosen": -0.9448979496955872,
+      "logits/rejected": -0.6800575256347656,
+      "logps/chosen": -294.2066345214844,
+      "logps/rejected": -319.7179870605469,
+      "loss": 0.1367,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.38026565313339233,
+      "rewards/margins": 0.39078813791275024,
+      "rewards/rejected": -0.7710537910461426,
       "step": 130
     },
     {
       "epoch": 0.3211009174311927,
-      "grad_norm": 25.0836527688945,
+      "grad_norm": 11.166916308293251,
       "learning_rate": 4.2958733752443187e-07,
-      "logits/chosen": -0.29172104597091675,
-      "logits/rejected": 0.11182677745819092,
-      "logps/chosen": -303.4015197753906,
-      "logps/rejected": -286.45526123046875,
-      "loss": 0.1334,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.2721634805202484,
-      "rewards/margins": 0.34942418336868286,
-      "rewards/rejected": -0.6215876340866089,
+      "logits/chosen": -0.8638193011283875,
+      "logits/rejected": -0.5810004472732544,
+      "logps/chosen": -307.26165771484375,
+      "logps/rejected": -287.42547607421875,
+      "loss": 0.134,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.3107650578022003,
+      "rewards/margins": 0.3205246925354004,
+      "rewards/rejected": -0.6312897801399231,
       "step": 140
     },
     {
       "epoch": 0.3440366972477064,
-      "grad_norm": 11.640337926409853,
+      "grad_norm": 10.491465778765697,
       "learning_rate": 4.150873668617898e-07,
-      "logits/chosen": 0.08257939666509628,
-      "logits/rejected": 0.5254168510437012,
-      "logps/chosen": -308.8121337890625,
-      "logps/rejected": -314.32220458984375,
-      "loss": 0.1307,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.42268529534339905,
-      "rewards/margins": 0.3886614143848419,
-      "rewards/rejected": -0.8113466501235962,
+      "logits/chosen": -0.785676121711731,
+      "logits/rejected": -0.4156356453895569,
+      "logps/chosen": -306.7540588378906,
+      "logps/rejected": -312.4991149902344,
+      "loss": 0.1327,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.40210476517677307,
+      "rewards/margins": 0.39101067185401917,
+      "rewards/rejected": -0.793115496635437,
       "step": 150
     },
     {
       "epoch": 0.3440366972477064,
-      "eval_logits/chosen": 0.018126631155610085,
-      "eval_logits/rejected": 0.7143937349319458,
-      "eval_logps/chosen": -320.6210021972656,
-      "eval_logps/rejected": -332.353271484375,
-      "eval_loss": 0.12613436579704285,
-      "eval_rewards/accuracies": 0.7284482717514038,
-      "eval_rewards/chosen": -0.3553086221218109,
-      "eval_rewards/margins": 0.5029994249343872,
-      "eval_rewards/rejected": -0.8583080768585205,
-      "eval_runtime": 91.5757,
-      "eval_samples_per_second": 19.852,
-      "eval_steps_per_second": 0.317,
+      "eval_logits/chosen": -0.934603214263916,
+      "eval_logits/rejected": -0.4213140308856964,
+      "eval_logps/chosen": -321.6920166015625,
+      "eval_logps/rejected": -333.3775939941406,
+      "eval_loss": 0.1256996989250183,
+      "eval_rewards/accuracies": 0.7241379022598267,
+      "eval_rewards/chosen": -0.36601918935775757,
+      "eval_rewards/margins": 0.5025321841239929,
+      "eval_rewards/rejected": -0.8685513734817505,
+      "eval_runtime": 95.7427,
+      "eval_samples_per_second": 18.988,
+      "eval_steps_per_second": 0.303,
       "step": 150
     },
     {
       "epoch": 0.3669724770642202,
-      "grad_norm": 11.56271576551216,
+      "grad_norm": 10.606572226202415,
       "learning_rate": 3.9952763262280397e-07,
-      "logits/chosen": 0.02932182513177395,
-      "logits/rejected": 0.4751170575618744,
-      "logps/chosen": -337.37237548828125,
-      "logps/rejected": -373.10382080078125,
-      "loss": 0.1247,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.40821242332458496,
-      "rewards/margins": 0.46313947439193726,
-      "rewards/rejected": -0.8713518381118774,
+      "logits/chosen": -0.6937485933303833,
+      "logits/rejected": -0.39156150817871094,
+      "logps/chosen": -341.2958068847656,
+      "logps/rejected": -376.64788818359375,
+      "loss": 0.1226,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.4474472105503082,
+      "rewards/margins": 0.4593452513217926,
+      "rewards/rejected": -0.9067924618721008,
       "step": 160
     },
     {
       "epoch": 0.38990825688073394,
-      "grad_norm": 7.985791463305948,
+      "grad_norm": 7.909933445936232,
       "learning_rate": 3.8300801912883414e-07,
-      "logits/chosen": 0.06568197906017303,
-      "logits/rejected": 0.3852883279323578,
-      "logps/chosen": -311.62481689453125,
-      "logps/rejected": -353.85443115234375,
-      "loss": 0.1191,
+      "logits/chosen": -0.5848881602287292,
+      "logits/rejected": -0.295288622379303,
+      "logps/chosen": -314.4745178222656,
+      "logps/rejected": -355.83209228515625,
+      "loss": 0.1194,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.40023931860923767,
-      "rewards/margins": 0.4376160204410553,
-      "rewards/rejected": -0.837855339050293,
+      "rewards/chosen": -0.42873620986938477,
+      "rewards/margins": 0.4288958013057709,
+      "rewards/rejected": -0.857632040977478,
       "step": 170
     },
     {
       "epoch": 0.41284403669724773,
-      "grad_norm": 9.33389793686436,
+      "grad_norm": 9.296204512324422,
       "learning_rate": 3.6563457256020884e-07,
-      "logits/chosen": 0.15779247879981995,
-      "logits/rejected": 0.6506751775741577,
-      "logps/chosen": -348.14398193359375,
-      "logps/rejected": -325.84637451171875,
-      "loss": 0.1278,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.44786080718040466,
-      "rewards/margins": 0.4256533682346344,
-      "rewards/rejected": -0.8735141754150391,
+      "logits/chosen": -0.4405382573604584,
+      "logits/rejected": 0.17199954390525818,
+      "logps/chosen": -351.5772399902344,
+      "logps/rejected": -332.4178771972656,
+      "loss": 0.1256,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.48219355940818787,
+      "rewards/margins": 0.4570356011390686,
+      "rewards/rejected": -0.9392291903495789,
       "step": 180
     },
     {
       "epoch": 0.43577981651376146,
-      "grad_norm": 10.313716932222244,
+      "grad_norm": 9.788859469537575,
       "learning_rate": 3.475188202022617e-07,
-      "logits/chosen": -0.43816322088241577,
-      "logits/rejected": 0.023333895951509476,
-      "logps/chosen": -283.8222351074219,
-      "logps/rejected": -337.9872741699219,
-      "loss": 0.1236,
+      "logits/chosen": -0.9616500735282898,
+      "logits/rejected": -0.5780771970748901,
+      "logps/chosen": -284.89788818359375,
+      "logps/rejected": -339.87139892578125,
+      "loss": 0.124,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.2570773959159851,
-      "rewards/margins": 0.47935089468955994,
-      "rewards/rejected": -0.7364283800125122,
+      "rewards/chosen": -0.2678338885307312,
+      "rewards/margins": 0.487435519695282,
+      "rewards/rejected": -0.7552694082260132,
       "step": 190
     },
     {
       "epoch": 0.45871559633027525,
-      "grad_norm": 9.248153089741166,
+      "grad_norm": 8.31792118291438,
       "learning_rate": 3.287770545059052e-07,
-      "logits/chosen": -0.08292710781097412,
-      "logits/rejected": 0.5214661359786987,
-      "logps/chosen": -312.56500244140625,
-      "logps/rejected": -325.7038269042969,
-      "loss": 0.1238,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.34283357858657837,
-      "rewards/margins": 0.4756447672843933,
-      "rewards/rejected": -0.8184784054756165,
+      "logits/chosen": -0.9191502332687378,
+      "logits/rejected": -0.3312894105911255,
+      "logps/chosen": -313.3896179199219,
+      "logps/rejected": -324.57000732421875,
+      "loss": 0.1242,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.3510800302028656,
+      "rewards/margins": 0.4560603201389313,
+      "rewards/rejected": -0.8071403503417969,
       "step": 200
     },
     {
       "epoch": 0.45871559633027525,
-      "eval_logits/chosen": 0.2969372272491455,
-      "eval_logits/rejected": 1.2989132404327393,
-      "eval_logps/chosen": -326.1716613769531,
-      "eval_logps/rejected": -341.28619384765625,
-      "eval_loss": 0.11991516500711441,
-      "eval_rewards/accuracies": 0.732758641242981,
-      "eval_rewards/chosen": -0.41081586480140686,
-      "eval_rewards/margins": 0.5368219614028931,
-      "eval_rewards/rejected": -0.9476377367973328,
-      "eval_runtime": 92.1824,
-      "eval_samples_per_second": 19.722,
-      "eval_steps_per_second": 0.315,
+      "eval_logits/chosen": -0.9084411859512329,
+      "eval_logits/rejected": -0.1497526615858078,
+      "eval_logps/chosen": -319.3252258300781,
+      "eval_logps/rejected": -331.5806884765625,
+      "eval_loss": 0.11925708502531052,
+      "eval_rewards/accuracies": 0.7456896305084229,
+      "eval_rewards/chosen": -0.34235113859176636,
+      "eval_rewards/margins": 0.5082312226295471,
+      "eval_rewards/rejected": -0.8505823612213135,
+      "eval_runtime": 93.7494,
+      "eval_samples_per_second": 19.392,
+      "eval_steps_per_second": 0.309,
       "step": 200
     },
     {
       "epoch": 0.481651376146789,
-      "grad_norm": 8.486691996298621,
+      "grad_norm": 7.584339731807936,
       "learning_rate": 3.0952958655864954e-07,
-      "logits/chosen": 0.504795253276825,
-      "logits/rejected": 1.0519931316375732,
-      "logps/chosen": -317.31756591796875,
-      "logps/rejected": -346.00689697265625,
-      "loss": 0.1175,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.44013166427612305,
-      "rewards/margins": 0.47651535272598267,
-      "rewards/rejected": -0.9166469573974609,
+      "logits/chosen": -0.6695399284362793,
+      "logits/rejected": -0.2294241189956665,
+      "logps/chosen": -311.20343017578125,
+      "logps/rejected": -338.2936096191406,
+      "loss": 0.1173,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.3789902925491333,
+      "rewards/margins": 0.4605233073234558,
+      "rewards/rejected": -0.8395135998725891,
       "step": 210
     },
     {
       "epoch": 0.5045871559633027,
-      "grad_norm": 9.587760322945808,
+      "grad_norm": 10.1245037946263,
       "learning_rate": 2.898999737583448e-07,
-      "logits/chosen": 0.056784339249134064,
-      "logits/rejected": 1.057108998298645,
-      "logps/chosen": -358.15313720703125,
-      "logps/rejected": -375.32452392578125,
-      "loss": 0.1134,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.3547281324863434,
-      "rewards/margins": 0.548815131187439,
-      "rewards/rejected": -0.9035432934761047,
+      "logits/chosen": -0.6778780221939087,
+      "logits/rejected": 0.15124213695526123,
+      "logps/chosen": -365.30010986328125,
+      "logps/rejected": -385.3759460449219,
+      "loss": 0.1146,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.42619770765304565,
+      "rewards/margins": 0.5778595209121704,
+      "rewards/rejected": -1.0040571689605713,
       "step": 220
     },
     {
       "epoch": 0.5275229357798165,
-      "grad_norm": 9.093354835969592,
+      "grad_norm": 8.611834221562093,
       "learning_rate": 2.7001422664752333e-07,
-      "logits/chosen": -0.12167753279209137,
-      "logits/rejected": 0.5305444002151489,
-      "logps/chosen": -298.165283203125,
-      "logps/rejected": -341.3959045410156,
-      "loss": 0.1206,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.3022175431251526,
-      "rewards/margins": 0.4555422365665436,
-      "rewards/rejected": -0.7577598094940186,
+      "logits/chosen": -0.5495241284370422,
+      "logits/rejected": -0.10778895765542984,
+      "logps/chosen": -308.42864990234375,
+      "logps/rejected": -350.29937744140625,
+      "loss": 0.1211,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.4048517346382141,
+      "rewards/margins": 0.4419426918029785,
+      "rewards/rejected": -0.8467944264411926,
       "step": 230
     },
     {
       "epoch": 0.5504587155963303,
-      "grad_norm": 9.439667913770359,
+      "grad_norm": 8.84744756928296,
       "learning_rate": 2.5e-07,
-      "logits/chosen": -0.21128520369529724,
-      "logits/rejected": 0.5969688296318054,
-      "logps/chosen": -321.5914306640625,
-      "logps/rejected": -339.0221862792969,
-      "loss": 0.1218,
+      "logits/chosen": -0.8606957197189331,
+      "logits/rejected": -0.25019243359565735,
+      "logps/chosen": -323.67840576171875,
+      "logps/rejected": -341.2455749511719,
+      "loss": 0.1198,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.34059566259384155,
-      "rewards/margins": 0.43612655997276306,
-      "rewards/rejected": -0.7767221927642822,
+      "rewards/chosen": -0.3614657521247864,
+      "rewards/margins": 0.4374900758266449,
+      "rewards/rejected": -0.7989557981491089,
       "step": 240
     },
     {
       "epoch": 0.573394495412844,
-      "grad_norm": 9.292342727311365,
+      "grad_norm": 8.762072273295908,
       "learning_rate": 2.2998577335247667e-07,
-      "logits/chosen": -0.020272482186555862,
-      "logits/rejected": 0.808585524559021,
-      "logps/chosen": -337.55767822265625,
-      "logps/rejected": -346.179443359375,
-      "loss": 0.1185,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.337638795375824,
-      "rewards/margins": 0.5197167992591858,
-      "rewards/rejected": -0.8573554754257202,
+      "logits/chosen": -0.8679742813110352,
+      "logits/rejected": -0.18818725645542145,
+      "logps/chosen": -340.2317810058594,
+      "logps/rejected": -349.81414794921875,
+      "loss": 0.118,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.3643796145915985,
+      "rewards/margins": 0.5293234586715698,
+      "rewards/rejected": -0.8937031030654907,
       "step": 250
     },
     {
       "epoch": 0.573394495412844,
-      "eval_logits/chosen": -0.1745007336139679,
-      "eval_logits/rejected": 0.8515735268592834,
-      "eval_logps/chosen": -315.9549560546875,
-      "eval_logps/rejected": -335.7632751464844,
-      "eval_loss": 0.11658623069524765,
+      "eval_logits/chosen": -0.8147245645523071,
+      "eval_logits/rejected": -0.025059914216399193,
+      "eval_logps/chosen": -322.4342041015625,
+      "eval_logps/rejected": -343.3682556152344,
+      "eval_loss": 0.11674168705940247,
       "eval_rewards/accuracies": 0.7543103694915771,
-      "eval_rewards/chosen": -0.3086487948894501,
-      "eval_rewards/margins": 0.5837592482566833,
-      "eval_rewards/rejected": -0.8924079537391663,
-      "eval_runtime": 92.3611,
-      "eval_samples_per_second": 19.684,
-      "eval_steps_per_second": 0.314,
+      "eval_rewards/chosen": -0.3734411895275116,
+      "eval_rewards/margins": 0.5950165390968323,
+      "eval_rewards/rejected": -0.9684576988220215,
+      "eval_runtime": 95.6284,
+      "eval_samples_per_second": 19.011,
+      "eval_steps_per_second": 0.303,
       "step": 250
     },
     {
       "epoch": 0.5963302752293578,
-      "grad_norm": 8.186794570934321,
+      "grad_norm": 8.055173398595795,
       "learning_rate": 2.1010002624165524e-07,
-      "logits/chosen": -0.006458556745201349,
-      "logits/rejected": 0.6227847933769226,
-      "logps/chosen": -308.7506408691406,
-      "logps/rejected": -372.9380798339844,
-      "loss": 0.1168,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.3803611397743225,
-      "rewards/margins": 0.6050828695297241,
-      "rewards/rejected": -0.9854438900947571,
+      "logits/chosen": -0.5792064666748047,
+      "logits/rejected": -0.028709029778838158,
+      "logps/chosen": -319.5377197265625,
+      "logps/rejected": -383.94610595703125,
+      "loss": 0.1186,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.4882320463657379,
+      "rewards/margins": 0.6072918176651001,
+      "rewards/rejected": -1.0955239534378052,
       "step": 260
     },
     {
       "epoch": 0.6192660550458715,
-      "grad_norm": 10.337683797583873,
+      "grad_norm": 9.235774055264796,
       "learning_rate": 1.9047041344135043e-07,
-      "logits/chosen": 0.44111576676368713,
-      "logits/rejected": 0.8292443156242371,
-      "logps/chosen": -313.4148254394531,
-      "logps/rejected": -354.33258056640625,
-      "loss": 0.1168,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.47352510690689087,
-      "rewards/margins": 0.5513299703598022,
-      "rewards/rejected": -1.0248548984527588,
+      "logits/chosen": -0.37728264927864075,
+      "logits/rejected": 0.017757948487997055,
+      "logps/chosen": -318.92266845703125,
+      "logps/rejected": -360.00250244140625,
+      "loss": 0.1169,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.5286036729812622,
+      "rewards/margins": 0.5529505014419556,
+      "rewards/rejected": -1.0815541744232178,
       "step": 270
     },
     {
       "epoch": 0.6422018348623854,
-      "grad_norm": 10.116957119119249,
+      "grad_norm": 9.286061693337006,
       "learning_rate": 1.7122294549409482e-07,
-      "logits/chosen": 0.1775570809841156,
-      "logits/rejected": 1.0465366840362549,
-      "logps/chosen": -317.04180908203125,
-      "logps/rejected": -370.72479248046875,
-      "loss": 0.1221,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.4404297471046448,
-      "rewards/margins": 0.6288600564002991,
-      "rewards/rejected": -1.0692898035049438,
+      "logits/chosen": -0.7753764986991882,
+      "logits/rejected": -0.038079630583524704,
+      "logps/chosen": -314.3120422363281,
+      "logps/rejected": -365.4776611328125,
+      "loss": 0.1216,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.4131316542625427,
+      "rewards/margins": 0.6036869883537292,
+      "rewards/rejected": -1.016818642616272,
       "step": 280
     },
     {
       "epoch": 0.6651376146788991,
-      "grad_norm": 9.609746165977107,
+      "grad_norm": 10.391059671359317,
       "learning_rate": 1.524811797977383e-07,
-      "logits/chosen": 0.02145857736468315,
-      "logits/rejected": 0.823032021522522,
-      "logps/chosen": -322.24542236328125,
-      "logps/rejected": -342.22503662109375,
-      "loss": 0.1156,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.3451458811759949,
-      "rewards/margins": 0.5379365086555481,
-      "rewards/rejected": -0.883082389831543,
+      "logits/chosen": -0.6743412017822266,
+      "logits/rejected": -0.05127597600221634,
+      "logps/chosen": -326.2575378417969,
+      "logps/rejected": -345.9309387207031,
+      "loss": 0.1144,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.3852667212486267,
+      "rewards/margins": 0.5348750352859497,
+      "rewards/rejected": -0.9201416969299316,
       "step": 290
     },
     {
       "epoch": 0.6880733944954128,
-      "grad_norm": 9.08082569283608,
+      "grad_norm": 11.142166437672913,
       "learning_rate": 1.3436542743979125e-07,
-      "logits/chosen": -0.18167677521705627,
-      "logits/rejected": 0.25138911604881287,
-      "logps/chosen": -341.6298828125,
-      "logps/rejected": -338.09869384765625,
-      "loss": 0.1228,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.3568056523799896,
-      "rewards/margins": 0.43558257818222046,
-      "rewards/rejected": -0.7923881411552429,
+      "logits/chosen": -0.6289922595024109,
+      "logits/rejected": -0.2742362916469574,
+      "logps/chosen": -350.96099853515625,
+      "logps/rejected": -345.9538879394531,
+      "loss": 0.1236,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.45011693239212036,
+      "rewards/margins": 0.42082324624061584,
+      "rewards/rejected": -0.8709400296211243,
       "step": 300
     },
     {
       "epoch": 0.6880733944954128,
-      "eval_logits/chosen": -0.13157324492931366,
-      "eval_logits/rejected": 0.8573818206787109,
-      "eval_logps/chosen": -322.0434265136719,
-      "eval_logps/rejected": -339.1875305175781,
-      "eval_loss": 0.11548212170600891,
-      "eval_rewards/accuracies": 0.7456896305084229,
-      "eval_rewards/chosen": -0.3695334494113922,
-      "eval_rewards/margins": 0.5571174621582031,
-      "eval_rewards/rejected": -0.9266510009765625,
-      "eval_runtime": 91.8257,
-      "eval_samples_per_second": 19.798,
-      "eval_steps_per_second": 0.316,
+      "eval_logits/chosen": -0.5284780263900757,
+      "eval_logits/rejected": 0.2890760600566864,
+      "eval_logps/chosen": -332.8935546875,
+      "eval_logps/rejected": -351.01617431640625,
+      "eval_loss": 0.11446455121040344,
+      "eval_rewards/accuracies": 0.7629310488700867,
+      "eval_rewards/chosen": -0.4780345857143402,
+      "eval_rewards/margins": 0.5669029355049133,
+      "eval_rewards/rejected": -1.0449376106262207,
+      "eval_runtime": 94.2008,
+      "eval_samples_per_second": 19.299,
+      "eval_steps_per_second": 0.308,
       "step": 300
     },
     {
       "epoch": 0.7110091743119266,
-      "grad_norm": 9.32642322141669,
+      "grad_norm": 9.249175988688211,
       "learning_rate": 1.1699198087116588e-07,
-      "logits/chosen": -0.07652176916599274,
-      "logits/rejected": 0.6259859800338745,
-      "logps/chosen": -321.20538330078125,
-      "logps/rejected": -355.15618896484375,
-      "loss": 0.1239,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.43240228295326233,
-      "rewards/margins": 0.4546627104282379,
-      "rewards/rejected": -0.887065052986145,
+      "logits/chosen": -0.5085445642471313,
+      "logits/rejected": 0.12936155498027802,
+      "logps/chosen": -332.48468017578125,
+      "logps/rejected": -366.32489013671875,
+      "loss": 0.1229,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5451949238777161,
+      "rewards/margins": 0.4535568654537201,
+      "rewards/rejected": -0.9987518191337585,
       "step": 310
     },
     {
       "epoch": 0.7339449541284404,
-      "grad_norm": 9.870138396091912,
+      "grad_norm": 9.319929398035963,
       "learning_rate": 1.00472367377196e-07,
-      "logits/chosen": 0.15437743067741394,
-      "logits/rejected": 0.9939621686935425,
-      "logps/chosen": -314.3326416015625,
-      "logps/rejected": -335.50787353515625,
-      "loss": 0.1193,
+      "logits/chosen": -0.37469834089279175,
+      "logits/rejected": 0.34990087151527405,
+      "logps/chosen": -323.309326171875,
+      "logps/rejected": -343.5829772949219,
+      "loss": 0.1184,
       "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.40100646018981934,
-      "rewards/margins": 0.6091704964637756,
-      "rewards/rejected": -1.0101768970489502,
+      "rewards/chosen": -0.49077337980270386,
+      "rewards/margins": 0.6001543402671814,
+      "rewards/rejected": -1.0909278392791748,
       "step": 320
     },
     {
       "epoch": 0.7568807339449541,
-      "grad_norm": 10.470680354911353,
+      "grad_norm": 10.87259656283017,
       "learning_rate": 8.49126331382102e-08,
-      "logits/chosen": 0.41432857513427734,
-      "logits/rejected": 0.9864773750305176,
-      "logps/chosen": -319.98114013671875,
-      "logps/rejected": -336.5364074707031,
-      "loss": 0.1183,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.45458975434303284,
-      "rewards/margins": 0.43977364897727966,
-      "rewards/rejected": -0.8943634033203125,
+      "logits/chosen": -0.20644304156303406,
+      "logits/rejected": 0.2470076084136963,
+      "logps/chosen": -329.26171875,
+      "logps/rejected": -346.26666259765625,
+      "loss": 0.1182,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5473949909210205,
+      "rewards/margins": 0.44427090883255005,
+      "rewards/rejected": -0.9916658401489258,
       "step": 330
     },
     {
       "epoch": 0.7798165137614679,
-      "grad_norm": 10.41854471188631,
+      "grad_norm": 11.502524720737265,
       "learning_rate": 7.041266247556812e-08,
-      "logits/chosen": 0.3841426968574524,
-      "logits/rejected": 0.9351291656494141,
-      "logps/chosen": -334.6560974121094,
-      "logps/rejected": -349.6623229980469,
-      "loss": 0.1152,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.4666348397731781,
-      "rewards/margins": 0.4301484227180481,
-      "rewards/rejected": -0.8967832326889038,
+      "logits/chosen": -0.511077344417572,
+      "logits/rejected": -0.07181791216135025,
+      "logps/chosen": -338.0530700683594,
+      "logps/rejected": -354.2392578125,
+      "loss": 0.1129,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.5006048679351807,
+      "rewards/margins": 0.4419478476047516,
+      "rewards/rejected": -0.9425527453422546,
       "step": 340
     },
     {
       "epoch": 0.8027522935779816,
-      "grad_norm": 9.996573814918259,
+      "grad_norm": 10.417951906388708,
       "learning_rate": 5.706553665319955e-08,
-      "logits/chosen": 0.6401292085647583,
-      "logits/rejected": 1.3996922969818115,
-      "logps/chosen": -324.6096496582031,
-      "logps/rejected": -337.38555908203125,
-      "loss": 0.1213,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.5011857151985168,
-      "rewards/margins": 0.4960555136203766,
-      "rewards/rejected": -0.997241199016571,
+      "logits/chosen": -0.4384083151817322,
+      "logits/rejected": 0.22936472296714783,
+      "logps/chosen": -327.1330261230469,
+      "logps/rejected": -339.122314453125,
+      "loss": 0.122,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.526419460773468,
+      "rewards/margins": 0.48818930983543396,
+      "rewards/rejected": -1.0146087408065796,
       "step": 350
     },
     {
       "epoch": 0.8027522935779816,
-      "eval_logits/chosen": 0.5151604413986206,
-      "eval_logits/rejected": 1.5740225315093994,
-      "eval_logps/chosen": -329.048583984375,
-      "eval_logps/rejected": -348.0972900390625,
-      "eval_loss": 0.113578200340271,
-      "eval_rewards/accuracies": 0.7629310488700867,
-      "eval_rewards/chosen": -0.4395850598812103,
-      "eval_rewards/margins": 0.5761635303497314,
-      "eval_rewards/rejected": -1.0157485008239746,
-      "eval_runtime": 92.2267,
-      "eval_samples_per_second": 19.712,
-      "eval_steps_per_second": 0.314,
+      "eval_logits/chosen": -0.48056575655937195,
+      "eval_logits/rejected": 0.33971619606018066,
+      "eval_logps/chosen": -330.8570251464844,
+      "eval_logps/rejected": -350.28302001953125,
+      "eval_loss": 0.11267489194869995,
+      "eval_rewards/accuracies": 0.767241358757019,
+      "eval_rewards/chosen": -0.45766934752464294,
+      "eval_rewards/margins": 0.5799364447593689,
+      "eval_rewards/rejected": -1.0376057624816895,
+      "eval_runtime": 94.5966,
+      "eval_samples_per_second": 19.218,
+      "eval_steps_per_second": 0.307,
       "step": 350
     },
     {
       "epoch": 0.8256880733944955,
-      "grad_norm": 8.23863269387286,
+      "grad_norm": 8.120253834632708,
       "learning_rate": 4.4956936350761005e-08,
-      "logits/chosen": 0.49835261702537537,
-      "logits/rejected": 1.0901247262954712,
-      "logps/chosen": -292.41424560546875,
-      "logps/rejected": -345.85540771484375,
-      "loss": 0.1131,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.40094178915023804,
-      "rewards/margins": 0.4722411632537842,
-      "rewards/rejected": -0.8731829524040222,
+      "logits/chosen": -0.412276029586792,
+      "logits/rejected": 0.03293975815176964,
+      "logps/chosen": -295.88397216796875,
+      "logps/rejected": -349.67828369140625,
+      "loss": 0.1126,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.43563956022262573,
+      "rewards/margins": 0.47577181458473206,
+      "rewards/rejected": -0.9114114046096802,
       "step": 360
     },
     {
       "epoch": 0.8486238532110092,
-      "grad_norm": 10.10305933177313,
+      "grad_norm": 8.937083414679394,
       "learning_rate": 3.416459164418123e-08,
-      "logits/chosen": 0.16921785473823547,
-      "logits/rejected": 0.9673768877983093,
-      "logps/chosen": -340.80059814453125,
-      "logps/rejected": -353.12274169921875,
-      "loss": 0.1129,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.41216129064559937,
-      "rewards/margins": 0.4933200776576996,
-      "rewards/rejected": -0.9054813385009766,
+      "logits/chosen": -0.7006546258926392,
+      "logits/rejected": -0.1087837815284729,
+      "logps/chosen": -344.0120544433594,
+      "logps/rejected": -357.1898498535156,
+      "loss": 0.1124,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.4442760944366455,
+      "rewards/margins": 0.5018762350082397,
+      "rewards/rejected": -0.9461522102355957,
       "step": 370
     },
     {
       "epoch": 0.8715596330275229,
-      "grad_norm": 9.150940276647933,
+      "grad_norm": 9.250415689561605,
       "learning_rate": 2.475778302439524e-08,
-      "logits/chosen": 0.3220874071121216,
-      "logits/rejected": 1.2661678791046143,
-      "logps/chosen": -332.2058410644531,
-      "logps/rejected": -351.96856689453125,
-      "loss": 0.1174,
+      "logits/chosen": -0.6538180112838745,
+      "logits/rejected": 0.11514680087566376,
+      "logps/chosen": -334.7962951660156,
+      "logps/rejected": -353.5735778808594,
+      "loss": 0.1165,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.35827815532684326,
-      "rewards/margins": 0.5848454236984253,
-      "rewards/rejected": -0.9431236386299133,
+      "rewards/chosen": -0.3841826617717743,
+      "rewards/margins": 0.5749909281730652,
+      "rewards/rejected": -0.9591735601425171,
       "step": 380
     },
     {
       "epoch": 0.8944954128440367,
-      "grad_norm": 9.488885903438641,
+      "grad_norm": 9.532136069213038,
       "learning_rate": 1.6796896657433805e-08,
-      "logits/chosen": 0.6681944727897644,
-      "logits/rejected": 1.2935806512832642,
-      "logps/chosen": -292.4158630371094,
-      "logps/rejected": -315.7539978027344,
-      "loss": 0.123,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.40458402037620544,
-      "rewards/margins": 0.4598938524723053,
-      "rewards/rejected": -0.8644779324531555,
+      "logits/chosen": -0.4097678065299988,
+      "logits/rejected": 0.13332059979438782,
+      "logps/chosen": -294.68280029296875,
+      "logps/rejected": -319.02825927734375,
+      "loss": 0.122,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.4272535443305969,
+      "rewards/margins": 0.4699665606021881,
+      "rewards/rejected": -0.8972201347351074,
       "step": 390
     },
     {
       "epoch": 0.9174311926605505,
-      "grad_norm": 9.000204295912624,
+      "grad_norm": 9.826819983657527,
       "learning_rate": 1.0333036740834855e-08,
-      "logits/chosen": 0.7832537889480591,
-      "logits/rejected": 1.2279746532440186,
-      "logps/chosen": -260.876708984375,
-      "logps/rejected": -314.25390625,
+      "logits/chosen": -0.3362836241722107,
+      "logits/rejected": 0.09532846510410309,
+      "logps/chosen": -262.51324462890625,
+      "logps/rejected": -315.52349853515625,
       "loss": 0.12,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.3631708323955536,
-      "rewards/margins": 0.4872121214866638,
-      "rewards/rejected": -0.8503829836845398,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.37953639030456543,
+      "rewards/margins": 0.48354291915893555,
+      "rewards/rejected": -0.8630792498588562,
       "step": 400
     },
     {
       "epoch": 0.9174311926605505,
-      "eval_logits/chosen": 0.40672120451927185,
-      "eval_logits/rejected": 1.4352502822875977,
-      "eval_logps/chosen": -324.5692443847656,
-      "eval_logps/rejected": -342.49981689453125,
-      "eval_loss": 0.11321888864040375,
-      "eval_rewards/accuracies": 0.7543103694915771,
-      "eval_rewards/chosen": -0.39479103684425354,
-      "eval_rewards/margins": 0.5649827122688293,
-      "eval_rewards/rejected": -0.9597737789154053,
-      "eval_runtime": 92.1464,
-      "eval_samples_per_second": 19.729,
-      "eval_steps_per_second": 0.315,
+      "eval_logits/chosen": -0.6159119606018066,
+      "eval_logits/rejected": 0.19038529694080353,
+      "eval_logps/chosen": -325.2745361328125,
+      "eval_logps/rejected": -342.9399108886719,
+      "eval_loss": 0.11256735771894455,
+      "eval_rewards/accuracies": 0.75,
+      "eval_rewards/chosen": -0.40184491872787476,
+      "eval_rewards/margins": 0.5623298287391663,
+      "eval_rewards/rejected": -0.9641746282577515,
+      "eval_runtime": 93.497,
+      "eval_samples_per_second": 19.444,
+      "eval_steps_per_second": 0.31,
       "step": 400
     },
     {
       "epoch": 0.9403669724770642,
-      "grad_norm": 12.797858678996972,
+      "grad_norm": 9.708164546519836,
       "learning_rate": 5.4076974448211685e-09,
-      "logits/chosen": 0.733371913433075,
-      "logits/rejected": 1.6556295156478882,
-      "logps/chosen": -313.32049560546875,
-      "logps/rejected": -336.0906677246094,
-      "loss": 0.1171,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.42368635535240173,
-      "rewards/margins": 0.5567531585693359,
-      "rewards/rejected": -0.9804395437240601,
+      "logits/chosen": -0.34120887517929077,
+      "logits/rejected": 0.403463214635849,
+      "logps/chosen": -314.7442321777344,
+      "logps/rejected": -337.3968811035156,
+      "loss": 0.1168,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.43792352080345154,
+      "rewards/margins": 0.5555781126022339,
+      "rewards/rejected": -0.9935015439987183,
       "step": 410
     },
     {
       "epoch": 0.963302752293578,
-      "grad_norm": 11.330514295442143,
+      "grad_norm": 10.889878191306106,
       "learning_rate": 2.052496544188487e-09,
-      "logits/chosen": 0.8590083122253418,
-      "logits/rejected": 1.5339264869689941,
-      "logps/chosen": -304.95831298828125,
-      "logps/rejected": -347.80169677734375,
-      "loss": 0.1153,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.4551934599876404,
-      "rewards/margins": 0.5640500783920288,
-      "rewards/rejected": -1.019243597984314,
+      "logits/chosen": -0.31919345259666443,
+      "logits/rejected": 0.3534119725227356,
+      "logps/chosen": -306.3693542480469,
+      "logps/rejected": -349.92462158203125,
+      "loss": 0.1147,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.4693039059638977,
+      "rewards/margins": 0.5711686015129089,
+      "rewards/rejected": -1.040472388267517,
       "step": 420
     },
     {
       "epoch": 0.9862385321100917,
-      "grad_norm": 9.086504274831077,
+      "grad_norm": 9.153719920278654,
       "learning_rate": 2.889724508297886e-10,
-      "logits/chosen": 0.5266289710998535,
-      "logits/rejected": 1.5435163974761963,
-      "logps/chosen": -344.6377868652344,
-      "logps/rejected": -336.5421447753906,
-      "loss": 0.1158,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.41462403535842896,
-      "rewards/margins": 0.5104321837425232,
-      "rewards/rejected": -0.9250561594963074,
+      "logits/chosen": -0.556784451007843,
+      "logits/rejected": 0.2818067669868469,
+      "logps/chosen": -345.05450439453125,
+      "logps/rejected": -335.64605712890625,
+      "loss": 0.1155,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.41879063844680786,
+      "rewards/margins": 0.49730420112609863,
+      "rewards/rejected": -0.9160947799682617,
       "step": 430
     },
     {
       "epoch": 1.0,
       "step": 436,
       "total_flos": 0.0,
-      "train_loss": 0.13218348616853767,
-      "train_runtime": 11343.5766,
-      "train_samples_per_second": 4.915,
+      "train_loss": 0.13197535372108493,
+      "train_runtime": 11523.0815,
+      "train_samples_per_second": 4.839,
       "train_steps_per_second": 0.038
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b97c68988ee761051dc175f846465ecfe6423242be1592fe1880198a31410e6
+oid sha256:3d6fef0b25cff78175dd3905d73113b24ac1239ca17c5ec75144eef77b24b3ac
 size 7608