Model save

README.md

@@ -4,6 +4,7 @@ base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: zephyr-7b-dpo-full-gpt_consistent-reward-scale-1-rpo
@@ -17,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0327
-- Rewards/chosen: -0.1258
-- Rewards/rejected: -0.4253
+- Loss: 0.1132
+- Rewards/chosen: -0.3948
+- Rewards/rejected: -0.9598
 - Rewards/accuracies: 0.7543
-- Rewards/margins: 0.2995
-- Logps/rejected: -289.0509
-- Logps/chosen: -297.6692
-- Logits/rejected: -1.4762
-- Logits/chosen: -1.6863
+- Rewards/margins: 0.5650
+- Logps/rejected: -342.4998
+- Logps/chosen: -324.5692
+- Logits/rejected: 1.4353
+- Logits/chosen: 0.4067
 
 ## Model description
 
@@ -62,14 +63,14 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0487        | 0.1147 | 50   | 0.0455          | 0.0300         | -0.0845          | 0.7026             | 0.1145          | -254.9693      | -282.0912    | -2.4928         | -2.5717       |
-| 0.0408        | 0.2294 | 100  | 0.0391          | -0.1071        | -0.3089          | 0.6897             | 0.2018          | -277.4089      | -295.8004    | -1.7416         | -1.8404       |
-| 0.0379        | 0.3440 | 150  | 0.0365          | -0.1406        | -0.4136          | 0.7155             | 0.2730          | -287.8874      | -299.1519    | -1.6357         | -1.7904       |
-| 0.0365        | 0.4587 | 200  | 0.0350          | -0.0650        | -0.3264          | 0.7543             | 0.2614          | -279.1631      | -291.5866    | -1.7552         | -1.9178       |
-| 0.0346        | 0.5734 | 250  | 0.0337          | -0.1319        | -0.4539          | 0.7543             | 0.3220          | -291.9156      | -298.2828    | -1.4871         | -1.7192       |
-| 0.036         | 0.6881 | 300  | 0.0331          | -0.1336        | -0.4291          | 0.75               | 0.2955          | -289.4286      | -298.4504    | -1.4842         | -1.6835       |
-| 0.0359        | 0.8028 | 350  | 0.0327          | -0.1378        | -0.4472          | 0.7586             | 0.3094          | -291.2399      | -298.8666    | -1.4658         | -1.6786       |
-| 0.0351        | 0.9174 | 400  | 0.0327          | -0.1258        | -0.4253          | 0.7543             | 0.2995          | -289.0509      | -297.6692    | -1.4762         | -1.6863       |
+| 0.1732        | 0.1147 | 50   | 0.1641          | 0.0166         | -0.1092          | 0.7069             | 0.1259          | -257.4446      | -283.4267    | -2.4943         | -2.5737       |
+| 0.1412        | 0.2294 | 100  | 0.1347          | -0.4342        | -0.7962          | 0.7284             | 0.3620          | -326.1406      | -328.5106    | -0.1538         | -0.4184       |
+| 0.1307        | 0.3440 | 150  | 0.1261          | -0.3553        | -0.8583          | 0.7284             | 0.5030          | -332.3533      | -320.6210    | 0.7144          | 0.0181        |
+| 0.1238        | 0.4587 | 200  | 0.1199          | -0.4108        | -0.9476          | 0.7328             | 0.5368          | -341.2862      | -326.1717    | 1.2989          | 0.2969        |
+| 0.1185        | 0.5734 | 250  | 0.1166          | -0.3086        | -0.8924          | 0.7543             | 0.5838          | -335.7633      | -315.9550    | 0.8516          | -0.1745       |
+| 0.1228        | 0.6881 | 300  | 0.1155          | -0.3695        | -0.9267          | 0.7457             | 0.5571          | -339.1875      | -322.0434    | 0.8574          | -0.1316       |
+| 0.1213        | 0.8028 | 350  | 0.1136          | -0.4396        | -1.0157          | 0.7629             | 0.5762          | -348.0973      | -329.0486    | 1.5740          | 0.5152        |
+| 0.12          | 0.9174 | 400  | 0.1132          | -0.3948        | -0.9598          | 0.7543             | 0.5650          | -342.4998      | -324.5692    | 1.4353          | 0.4067        |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,22 @@
 {
     "epoch": 1.0,
+    "eval_logits/chosen": -1.684370994567871,
+    "eval_logits/rejected": -1.4736462831497192,
+    "eval_logps/chosen": -297.6668701171875,
+    "eval_logps/rejected": -289.2203674316406,
+    "eval_loss": 0.032715052366256714,
+    "eval_rewards/accuracies": 0.7543103694915771,
+    "eval_rewards/chosen": -0.1257677674293518,
+    "eval_rewards/margins": 0.30121153593063354,
+    "eval_rewards/rejected": -0.42697927355766296,
+    "eval_runtime": 94.8006,
+    "eval_samples": 1818,
+    "eval_samples_per_second": 19.177,
+    "eval_steps_per_second": 0.306,
     "total_flos": 0.0,
-    "train_loss": 0.0384325077500912,
-    "train_runtime": 11659.9634,
+    "train_loss": 0.13218348616853767,
+    "train_runtime": 11343.5766,
     "train_samples": 55758,
-    "train_samples_per_second": 4.782,
-    "train_steps_per_second": 0.037
+    "train_samples_per_second": 4.915,
+    "train_steps_per_second": 0.038
 }

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -1.684370994567871,
+    "eval_logits/rejected": -1.4736462831497192,
+    "eval_logps/chosen": -297.6668701171875,
+    "eval_logps/rejected": -289.2203674316406,
+    "eval_loss": 0.032715052366256714,
+    "eval_rewards/accuracies": 0.7543103694915771,
+    "eval_rewards/chosen": -0.1257677674293518,
+    "eval_rewards/margins": 0.30121153593063354,
+    "eval_rewards/rejected": -0.42697927355766296,
+    "eval_runtime": 94.8006,
+    "eval_samples": 1818,
+    "eval_samples_per_second": 19.177,
+    "eval_steps_per_second": 0.306
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de11b87922b4b44c8cca13b4fd1512d1d8080e08297e93d938d39bd61aa84066
+oid sha256:5905b6fface4e45339623c4d375f9edc78e435e73d598309e1ed43118ed92014
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57aaa71d01065cadbe8c8c60ab7a99a32cc956d53ba264fbbe7c076dfe1f5471
+oid sha256:b04ddbeebeda55ae3848556f3a71f59213ac25a02039521f54874d8cdbff36f1
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dcfba798eaadf6d113a0642a66150398a3a3378d41bce7656fb7e6e2eff751f1
+oid sha256:b01821c5089f266496aec2a04ad6bae11924a4608f9a7968429e02dcfedff8ea
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.0384325077500912,
-    "train_runtime": 11659.9634,
+    "train_loss": 0.13218348616853767,
+    "train_runtime": 11343.5766,
     "train_samples": 55758,
-    "train_samples_per_second": 4.782,
-    "train_steps_per_second": 0.037
+    "train_samples_per_second": 4.915,
+    "train_steps_per_second": 0.038
 }

trainer_state.json

@@ -10,785 +10,785 @@
   "log_history": [
     {
       "epoch": 0.022935779816513763,
-      "grad_norm": 3.1641646757229265,
+      "grad_norm": 5.597671583305064,
       "learning_rate": 1.1363636363636363e-07,
-      "logits/chosen": -2.6194896697998047,
-      "logits/rejected": -2.552656650543213,
-      "logps/chosen": -265.41778564453125,
-      "logps/rejected": -236.15280151367188,
-      "loss": 0.0585,
-      "rewards/accuracies": 0.39375001192092896,
-      "rewards/chosen": 0.00028036616276949644,
-      "rewards/margins": 0.0005085940356366336,
-      "rewards/rejected": -0.00022822784376330674,
+      "logits/chosen": -2.6193342208862305,
+      "logits/rejected": -2.5525386333465576,
+      "logps/chosen": -265.3854064941406,
+      "logps/rejected": -236.1589813232422,
+      "loss": 0.1941,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": 0.0006044252077117562,
+      "rewards/margins": 0.0008946189773268998,
+      "rewards/rejected": -0.000290193798718974,
       "step": 10
     },
     {
       "epoch": 0.045871559633027525,
-      "grad_norm": 2.9506602825308903,
+      "grad_norm": 5.252146327563864,
       "learning_rate": 2.2727272727272726e-07,
-      "logits/chosen": -2.6579580307006836,
-      "logits/rejected": -2.575996160507202,
-      "logps/chosen": -298.8293151855469,
-      "logps/rejected": -274.3349304199219,
-      "loss": 0.059,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.0004406004154589027,
-      "rewards/margins": 0.0018091611564159393,
-      "rewards/rejected": -0.0022497617173939943,
+      "logits/chosen": -2.657933473587036,
+      "logits/rejected": -2.576028347015381,
+      "logps/chosen": -298.8043518066406,
+      "logps/rejected": -274.30718994140625,
+      "loss": 0.1958,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.00019080772472079843,
+      "rewards/margins": 0.0017816700274124742,
+      "rewards/rejected": -0.001972477650269866,
       "step": 20
     },
     {
       "epoch": 0.06880733944954129,
-      "grad_norm": 2.7648586784794067,
+      "grad_norm": 4.974318928321663,
       "learning_rate": 3.4090909090909085e-07,
-      "logits/chosen": -2.6760480403900146,
-      "logits/rejected": -2.6023683547973633,
-      "logps/chosen": -290.41986083984375,
-      "logps/rejected": -234.40579223632812,
-      "loss": 0.0571,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.009129038080573082,
-      "rewards/margins": 0.01336810551583767,
-      "rewards/rejected": -0.004239068366587162,
+      "logits/chosen": -2.6759729385375977,
+      "logits/rejected": -2.6023097038269043,
+      "logps/chosen": -290.3868713378906,
+      "logps/rejected": -234.3571319580078,
+      "loss": 0.1923,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.009458948858082294,
+      "rewards/margins": 0.013211230747401714,
+      "rewards/rejected": -0.0037522814236581326,
       "step": 30
     },
     {
       "epoch": 0.09174311926605505,
-      "grad_norm": 2.7634574887860825,
+      "grad_norm": 5.209557025843711,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": -2.6598825454711914,
-      "logits/rejected": -2.610196352005005,
-      "logps/chosen": -280.923828125,
-      "logps/rejected": -267.7273864746094,
-      "loss": 0.0529,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.04245365411043167,
-      "rewards/margins": 0.04179743677377701,
-      "rewards/rejected": 0.0006562232738360763,
+      "logits/chosen": -2.659982204437256,
+      "logits/rejected": -2.6102194786071777,
+      "logps/chosen": -280.9975891113281,
+      "logps/rejected": -267.7278747558594,
+      "loss": 0.1836,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04171619936823845,
+      "rewards/margins": 0.041064582765102386,
+      "rewards/rejected": 0.0006516153225675225,
       "step": 40
     },
     {
       "epoch": 0.11467889908256881,
-      "grad_norm": 2.9044861722207567,
+      "grad_norm": 5.582751836908906,
       "learning_rate": 4.997110275491701e-07,
-      "logits/chosen": -2.620276927947998,
-      "logits/rejected": -2.6136012077331543,
-      "logps/chosen": -292.2350158691406,
-      "logps/rejected": -302.38018798828125,
-      "loss": 0.0487,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.04016200453042984,
-      "rewards/margins": 0.07212186604738235,
-      "rewards/rejected": -0.031959857791662216,
+      "logits/chosen": -2.6198437213897705,
+      "logits/rejected": -2.6123862266540527,
+      "logps/chosen": -293.4254455566406,
+      "logps/rejected": -303.90386962890625,
+      "loss": 0.1732,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.028257910162210464,
+      "rewards/margins": 0.0754549652338028,
+      "rewards/rejected": -0.047197047621011734,
       "step": 50
     },
     {
       "epoch": 0.11467889908256881,
-      "eval_logits/chosen": -2.5716989040374756,
-      "eval_logits/rejected": -2.4927897453308105,
-      "eval_logps/chosen": -282.0911865234375,
-      "eval_logps/rejected": -254.96926879882812,
-      "eval_loss": 0.04548575356602669,
-      "eval_rewards/accuracies": 0.7025862336158752,
-      "eval_rewards/chosen": 0.02998923696577549,
-      "eval_rewards/margins": 0.11445755511522293,
-      "eval_rewards/rejected": -0.08446833491325378,
-      "eval_runtime": 95.7446,
-      "eval_samples_per_second": 18.988,
-      "eval_steps_per_second": 0.303,
+      "eval_logits/chosen": -2.5737149715423584,
+      "eval_logits/rejected": -2.494300365447998,
+      "eval_logps/chosen": -283.4266662597656,
+      "eval_logps/rejected": -257.4446105957031,
+      "eval_loss": 0.16405636072158813,
+      "eval_rewards/accuracies": 0.7068965435028076,
+      "eval_rewards/chosen": 0.016634328290820122,
+      "eval_rewards/margins": 0.12585583329200745,
+      "eval_rewards/rejected": -0.10922150313854218,
+      "eval_runtime": 93.7026,
+      "eval_samples_per_second": 19.402,
+      "eval_steps_per_second": 0.309,
       "step": 50
     },
     {
       "epoch": 0.13761467889908258,
-      "grad_norm": 3.2642316994426857,
+      "grad_norm": 7.241200335566244,
       "learning_rate": 4.979475034558115e-07,
-      "logits/chosen": -2.5661163330078125,
-      "logits/rejected": -2.50827956199646,
-      "logps/chosen": -289.7759704589844,
-      "logps/rejected": -267.379638671875,
-      "loss": 0.0449,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.0029130387119948864,
-      "rewards/margins": 0.09531786292791367,
-      "rewards/rejected": -0.09823091328144073,
+      "logits/chosen": -2.562969207763672,
+      "logits/rejected": -2.5053610801696777,
+      "logps/chosen": -293.1821594238281,
+      "logps/rejected": -272.9414367675781,
+      "loss": 0.1593,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03697499260306358,
+      "rewards/margins": 0.11687393486499786,
+      "rewards/rejected": -0.15384894609451294,
       "step": 60
     },
     {
       "epoch": 0.16055045871559634,
-      "grad_norm": 3.167167289379583,
+      "grad_norm": 7.058665086800562,
       "learning_rate": 4.945923025551788e-07,
-      "logits/chosen": -2.460839033126831,
-      "logits/rejected": -2.4139201641082764,
-      "logps/chosen": -324.13690185546875,
-      "logps/rejected": -276.5367126464844,
-      "loss": 0.0446,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.034340955317020416,
-      "rewards/margins": 0.1856319159269333,
-      "rewards/rejected": -0.15129096806049347,
+      "logits/chosen": -2.4592204093933105,
+      "logits/rejected": -2.4099252223968506,
+      "logps/chosen": -327.544189453125,
+      "logps/rejected": -284.48760986328125,
+      "loss": 0.1564,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.00026793667348101735,
+      "rewards/margins": 0.23106786608695984,
+      "rewards/rejected": -0.23079994320869446,
       "step": 70
     },
     {
       "epoch": 0.1834862385321101,
-      "grad_norm": 3.00053019606861,
+      "grad_norm": 8.05919017424508,
       "learning_rate": 4.896669632591651e-07,
-      "logits/chosen": -2.356163740158081,
-      "logits/rejected": -2.2467029094696045,
-      "logps/chosen": -287.4019470214844,
-      "logps/rejected": -272.3543701171875,
-      "loss": 0.0416,
+      "logits/chosen": -2.3360755443573,
+      "logits/rejected": -2.2115492820739746,
+      "logps/chosen": -298.69482421875,
+      "logps/rejected": -292.111328125,
+      "loss": 0.1478,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.015520763583481312,
-      "rewards/margins": 0.18146219849586487,
-      "rewards/rejected": -0.1969829499721527,
+      "rewards/chosen": -0.128449946641922,
+      "rewards/margins": 0.26610296964645386,
+      "rewards/rejected": -0.39455294609069824,
       "step": 80
     },
     {
       "epoch": 0.20642201834862386,
-      "grad_norm": 3.936347429433164,
+      "grad_norm": 8.402102216426163,
       "learning_rate": 4.832031033425662e-07,
-      "logits/chosen": -2.0837090015411377,
-      "logits/rejected": -2.0328478813171387,
-      "logps/chosen": -287.70654296875,
-      "logps/rejected": -268.2559509277344,
-      "loss": 0.0405,
+      "logits/chosen": -1.9170547723770142,
+      "logits/rejected": -1.8550583124160767,
+      "logps/chosen": -299.20281982421875,
+      "logps/rejected": -289.49163818359375,
+      "loss": 0.1441,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.001131159020587802,
-      "rewards/margins": 0.2134155035018921,
-      "rewards/rejected": -0.21454665064811707,
+      "rewards/chosen": -0.1160941943526268,
+      "rewards/margins": 0.3108092248439789,
+      "rewards/rejected": -0.4269034266471863,
       "step": 90
     },
     {
       "epoch": 0.22935779816513763,
-      "grad_norm": 3.652555898125068,
+      "grad_norm": 8.625504630985954,
       "learning_rate": 4.752422169756047e-07,
-      "logits/chosen": -1.5697656869888306,
-      "logits/rejected": -1.561859130859375,
-      "logps/chosen": -289.6307678222656,
-      "logps/rejected": -307.66363525390625,
-      "loss": 0.0408,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.17206048965454102,
-      "rewards/margins": 0.18529178202152252,
-      "rewards/rejected": -0.35735225677490234,
+      "logits/chosen": -0.48900121450424194,
+      "logits/rejected": -0.44961875677108765,
+      "logps/chosen": -311.1794738769531,
+      "logps/rejected": -339.92254638671875,
+      "loss": 0.1412,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.3875480592250824,
+      "rewards/margins": 0.2923927903175354,
+      "rewards/rejected": -0.6799408197402954,
       "step": 100
     },
     {
       "epoch": 0.22935779816513763,
-      "eval_logits/chosen": -1.8404057025909424,
-      "eval_logits/rejected": -1.7416090965270996,
-      "eval_logps/chosen": -295.8004455566406,
-      "eval_logps/rejected": -277.408935546875,
-      "eval_loss": 0.03911930322647095,
-      "eval_rewards/accuracies": 0.6896551847457886,
-      "eval_rewards/chosen": -0.10710364580154419,
-      "eval_rewards/margins": 0.201760932803154,
-      "eval_rewards/rejected": -0.308864563703537,
-      "eval_runtime": 95.267,
-      "eval_samples_per_second": 19.083,
-      "eval_steps_per_second": 0.304,
+      "eval_logits/chosen": -0.4184306561946869,
+      "eval_logits/rejected": -0.1537734419107437,
+      "eval_logps/chosen": -328.5106201171875,
+      "eval_logps/rejected": -326.1405944824219,
+      "eval_loss": 0.13468901813030243,
+      "eval_rewards/accuracies": 0.7284482717514038,
+      "eval_rewards/chosen": -0.4342052936553955,
+      "eval_rewards/margins": 0.3619759678840637,
+      "eval_rewards/rejected": -0.796181321144104,
+      "eval_runtime": 92.0893,
+      "eval_samples_per_second": 19.742,
+      "eval_steps_per_second": 0.315,
       "step": 100
     },
     {
       "epoch": 0.25229357798165136,
-      "grad_norm": 3.509676092042844,
+      "grad_norm": 10.103043851045701,
       "learning_rate": 4.658354083558188e-07,
-      "logits/chosen": -1.9262816905975342,
-      "logits/rejected": -1.80355966091156,
-      "logps/chosen": -279.3715515136719,
-      "logps/rejected": -263.81549072265625,
-      "loss": 0.0402,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.14183419942855835,
-      "rewards/margins": 0.15898862481117249,
-      "rewards/rejected": -0.30082279443740845,
+      "logits/chosen": -0.40221118927001953,
+      "logits/rejected": -0.13967064023017883,
+      "logps/chosen": -312.624267578125,
+      "logps/rejected": -311.3497009277344,
+      "loss": 0.1354,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.4743613302707672,
+      "rewards/margins": 0.3018040060997009,
+      "rewards/rejected": -0.776165246963501,
       "step": 110
     },
     {
       "epoch": 0.27522935779816515,
-      "grad_norm": 3.984377752564919,
+      "grad_norm": 10.671011000324334,
       "learning_rate": 4.550430636492389e-07,
-      "logits/chosen": -1.8143861293792725,
-      "logits/rejected": -1.7607357501983643,
-      "logps/chosen": -295.43243408203125,
-      "logps/rejected": -289.3270568847656,
-      "loss": 0.0395,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.19613118469715118,
-      "rewards/margins": 0.18250016868114471,
-      "rewards/rejected": -0.3786313831806183,
+      "logits/chosen": -0.04719700291752815,
+      "logits/rejected": 0.09439365565776825,
+      "logps/chosen": -324.4268798828125,
+      "logps/rejected": -335.16424560546875,
+      "loss": 0.1353,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4860754609107971,
+      "rewards/margins": 0.3509276807308197,
+      "rewards/rejected": -0.8370031118392944,
       "step": 120
     },
     {
       "epoch": 0.2981651376146789,
-      "grad_norm": 3.895538046894766,
+      "grad_norm": 11.348522180816168,
       "learning_rate": 4.429344633468004e-07,
-      "logits/chosen": -1.742222785949707,
-      "logits/rejected": -1.6753323078155518,
-      "logps/chosen": -268.340087890625,
-      "logps/rejected": -275.31500244140625,
-      "loss": 0.0396,
+      "logits/chosen": -0.058751799166202545,
+      "logits/rejected": 0.29970529675483704,
+      "logps/chosen": -296.40399169921875,
+      "logps/rejected": -316.8427429199219,
+      "loss": 0.1373,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.12159997224807739,
-      "rewards/margins": 0.20542438328266144,
-      "rewards/rejected": -0.32702434062957764,
+      "rewards/chosen": -0.4022388458251953,
+      "rewards/margins": 0.3400627374649048,
+      "rewards/rejected": -0.7423015832901001,
       "step": 130
     },
     {
       "epoch": 0.3211009174311927,
-      "grad_norm": 4.968671690125824,
+      "grad_norm": 25.0836527688945,
       "learning_rate": 4.2958733752443187e-07,
-      "logits/chosen": -1.6684496402740479,
-      "logits/rejected": -1.5686622858047485,
-      "logps/chosen": -286.41973876953125,
-      "logps/rejected": -254.42691040039062,
-      "loss": 0.0381,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.10234608501195908,
-      "rewards/margins": 0.19895747303962708,
-      "rewards/rejected": -0.30130359530448914,
+      "logits/chosen": -0.29172104597091675,
+      "logits/rejected": 0.11182677745819092,
+      "logps/chosen": -303.4015197753906,
+      "logps/rejected": -286.45526123046875,
+      "loss": 0.1334,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2721634805202484,
+      "rewards/margins": 0.34942418336868286,
+      "rewards/rejected": -0.6215876340866089,
       "step": 140
     },
     {
       "epoch": 0.3440366972477064,
-      "grad_norm": 3.5584001205811377,
+      "grad_norm": 11.640337926409853,
       "learning_rate": 4.150873668617898e-07,
-      "logits/chosen": -1.703791856765747,
-      "logits/rejected": -1.5906720161437988,
-      "logps/chosen": -280.5138854980469,
-      "logps/rejected": -269.3014831542969,
-      "loss": 0.0379,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.13970300555229187,
-      "rewards/margins": 0.22143635153770447,
-      "rewards/rejected": -0.36113935708999634,
+      "logits/chosen": 0.08257939666509628,
+      "logits/rejected": 0.5254168510437012,
+      "logps/chosen": -308.8121337890625,
+      "logps/rejected": -314.32220458984375,
+      "loss": 0.1307,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.42268529534339905,
+      "rewards/margins": 0.3886614143848419,
+      "rewards/rejected": -0.8113466501235962,
       "step": 150
     },
     {
       "epoch": 0.3440366972477064,
-      "eval_logits/chosen": -1.7903783321380615,
-      "eval_logits/rejected": -1.635672688484192,
-      "eval_logps/chosen": -299.1519470214844,
-      "eval_logps/rejected": -287.88739013671875,
-      "eval_loss": 0.03649381175637245,
-      "eval_rewards/accuracies": 0.7155172228813171,
-      "eval_rewards/chosen": -0.14061833918094635,
-      "eval_rewards/margins": 0.27303099632263184,
-      "eval_rewards/rejected": -0.413649320602417,
-      "eval_runtime": 94.6259,
-      "eval_samples_per_second": 19.212,
-      "eval_steps_per_second": 0.306,
+      "eval_logits/chosen": 0.018126631155610085,
+      "eval_logits/rejected": 0.7143937349319458,
+      "eval_logps/chosen": -320.6210021972656,
+      "eval_logps/rejected": -332.353271484375,
+      "eval_loss": 0.12613436579704285,
+      "eval_rewards/accuracies": 0.7284482717514038,
+      "eval_rewards/chosen": -0.3553086221218109,
+      "eval_rewards/margins": 0.5029994249343872,
+      "eval_rewards/rejected": -0.8583080768585205,
+      "eval_runtime": 91.5757,
+      "eval_samples_per_second": 19.852,
+      "eval_steps_per_second": 0.317,
       "step": 150
     },
     {
       "epoch": 0.3669724770642202,
-      "grad_norm": 5.2149351876346905,
+      "grad_norm": 11.56271576551216,
       "learning_rate": 3.9952763262280397e-07,
-      "logits/chosen": -1.6572643518447876,
-      "logits/rejected": -1.5417451858520508,
-      "logps/chosen": -309.5204162597656,
-      "logps/rejected": -324.3457946777344,
-      "loss": 0.0359,
+      "logits/chosen": 0.02932182513177395,
+      "logits/rejected": 0.4751170575618744,
+      "logps/chosen": -337.37237548828125,
+      "logps/rejected": -373.10382080078125,
+      "loss": 0.1247,
       "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.12969312071800232,
-      "rewards/margins": 0.254078209400177,
-      "rewards/rejected": -0.3837713599205017,
+      "rewards/chosen": -0.40821242332458496,
+      "rewards/margins": 0.46313947439193726,
+      "rewards/rejected": -0.8713518381118774,
       "step": 160
     },
     {
       "epoch": 0.38990825688073394,
-      "grad_norm": 2.9551901080690395,
+      "grad_norm": 7.985791463305948,
       "learning_rate": 3.8300801912883414e-07,
-      "logits/chosen": -1.6375774145126343,
-      "logits/rejected": -1.5148413181304932,
-      "logps/chosen": -277.994873046875,
-      "logps/rejected": -301.9854431152344,
-      "loss": 0.0339,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.06393978744745255,
-      "rewards/margins": 0.2552258372306824,
-      "rewards/rejected": -0.3191656172275543,
+      "logits/chosen": 0.06568197906017303,
+      "logits/rejected": 0.3852883279323578,
+      "logps/chosen": -311.62481689453125,
+      "logps/rejected": -353.85443115234375,
+      "loss": 0.1191,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.40023931860923767,
+      "rewards/margins": 0.4376160204410553,
+      "rewards/rejected": -0.837855339050293,
       "step": 170
     },
     {
       "epoch": 0.41284403669724773,
-      "grad_norm": 3.6761291533851423,
+      "grad_norm": 9.33389793686436,
       "learning_rate": 3.6563457256020884e-07,
-      "logits/chosen": -1.527038335800171,
-      "logits/rejected": -1.2771427631378174,
-      "logps/chosen": -323.6756896972656,
-      "logps/rejected": -284.1974792480469,
-      "loss": 0.0365,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.20317766070365906,
-      "rewards/margins": 0.2538478374481201,
-      "rewards/rejected": -0.4570254683494568,
+      "logits/chosen": 0.15779247879981995,
+      "logits/rejected": 0.6506751775741577,
+      "logps/chosen": -348.14398193359375,
+      "logps/rejected": -325.84637451171875,
+      "loss": 0.1278,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.44786080718040466,
+      "rewards/margins": 0.4256533682346344,
+      "rewards/rejected": -0.8735141754150391,
       "step": 180
     },
     {
       "epoch": 0.43577981651376146,
-      "grad_norm": 5.675419312165092,
+      "grad_norm": 10.313716932222244,
       "learning_rate": 3.475188202022617e-07,
-      "logits/chosen": -1.6745436191558838,
-      "logits/rejected": -1.604264259338379,
-      "logps/chosen": -268.5342102050781,
-      "logps/rejected": -302.51898193359375,
-      "loss": 0.0368,
+      "logits/chosen": -0.43816322088241577,
+      "logits/rejected": 0.023333895951509476,
+      "logps/chosen": -283.8222351074219,
+      "logps/rejected": -337.9872741699219,
+      "loss": 0.1236,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.10419674217700958,
-      "rewards/margins": 0.27754858136177063,
-      "rewards/rejected": -0.38174527883529663,
+      "rewards/chosen": -0.2570773959159851,
+      "rewards/margins": 0.47935089468955994,
+      "rewards/rejected": -0.7364283800125122,
       "step": 190
     },
     {
       "epoch": 0.45871559633027525,
-      "grad_norm": 3.4589111733943283,
+      "grad_norm": 9.248153089741166,
       "learning_rate": 3.287770545059052e-07,
-      "logits/chosen": -1.8974136114120483,
-      "logits/rejected": -1.7577918767929077,
-      "logps/chosen": -291.4000549316406,
-      "logps/rejected": -281.4195861816406,
-      "loss": 0.0365,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.1311841905117035,
-      "rewards/margins": 0.24445155262947083,
-      "rewards/rejected": -0.3756357431411743,
+      "logits/chosen": -0.08292710781097412,
+      "logits/rejected": 0.5214661359786987,
+      "logps/chosen": -312.56500244140625,
+      "logps/rejected": -325.7038269042969,
+      "loss": 0.1238,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.34283357858657837,
+      "rewards/margins": 0.4756447672843933,
+      "rewards/rejected": -0.8184784054756165,
       "step": 200
     },
     {
       "epoch": 0.45871559633027525,
-      "eval_logits/chosen": -1.9178062677383423,
-      "eval_logits/rejected": -1.7552225589752197,
-      "eval_logps/chosen": -291.5865783691406,
-      "eval_logps/rejected": -279.1630859375,
-      "eval_loss": 0.03504867106676102,
-      "eval_rewards/accuracies": 0.7543103694915771,
-      "eval_rewards/chosen": -0.06496486067771912,
-      "eval_rewards/margins": 0.2614416182041168,
-      "eval_rewards/rejected": -0.32640647888183594,
-      "eval_runtime": 95.5023,
-      "eval_samples_per_second": 19.036,
-      "eval_steps_per_second": 0.304,
+      "eval_logits/chosen": 0.2969372272491455,
+      "eval_logits/rejected": 1.2989132404327393,
+      "eval_logps/chosen": -326.1716613769531,
+      "eval_logps/rejected": -341.28619384765625,
+      "eval_loss": 0.11991516500711441,
+      "eval_rewards/accuracies": 0.732758641242981,
+      "eval_rewards/chosen": -0.41081586480140686,
+      "eval_rewards/margins": 0.5368219614028931,
+      "eval_rewards/rejected": -0.9476377367973328,
+      "eval_runtime": 92.1824,
+      "eval_samples_per_second": 19.722,
+      "eval_steps_per_second": 0.315,
       "step": 200
     },
     {
       "epoch": 0.481651376146789,
-      "grad_norm": 2.8163994777137717,
+      "grad_norm": 8.486691996298621,
       "learning_rate": 3.0952958655864954e-07,
-      "logits/chosen": -1.794647455215454,
-      "logits/rejected": -1.7260096073150635,
-      "logps/chosen": -280.9212951660156,
-      "logps/rejected": -287.47674560546875,
-      "loss": 0.0349,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.07616890966892242,
-      "rewards/margins": 0.2551766633987427,
-      "rewards/rejected": -0.3313455581665039,
+      "logits/chosen": 0.504795253276825,
+      "logits/rejected": 1.0519931316375732,
+      "logps/chosen": -317.31756591796875,
+      "logps/rejected": -346.00689697265625,
+      "loss": 0.1175,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.44013166427612305,
+      "rewards/margins": 0.47651535272598267,
+      "rewards/rejected": -0.9166469573974609,
       "step": 210
     },
     {
       "epoch": 0.5045871559633027,
-      "grad_norm": 3.602353089915511,
+      "grad_norm": 9.587760322945808,
       "learning_rate": 2.898999737583448e-07,
-      "logits/chosen": -1.672721266746521,
-      "logits/rejected": -1.4848198890686035,
-      "logps/chosen": -344.14825439453125,
-      "logps/rejected": -337.38470458984375,
-      "loss": 0.0336,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.2146792709827423,
-      "rewards/margins": 0.3094654977321625,
-      "rewards/rejected": -0.5241447687149048,
+      "logits/chosen": 0.056784339249134064,
+      "logits/rejected": 1.057108998298645,
+      "logps/chosen": -358.15313720703125,
+      "logps/rejected": -375.32452392578125,
+      "loss": 0.1134,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.3547281324863434,
+      "rewards/margins": 0.548815131187439,
+      "rewards/rejected": -0.9035432934761047,
       "step": 220
     },
     {
       "epoch": 0.5275229357798165,
-      "grad_norm": 3.2287234258499633,
+      "grad_norm": 9.093354835969592,
       "learning_rate": 2.7001422664752333e-07,
-      "logits/chosen": -1.4607375860214233,
-      "logits/rejected": -1.3907452821731567,
-      "logps/chosen": -288.92095947265625,
-      "logps/rejected": -309.4497985839844,
-      "loss": 0.0354,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.20977428555488586,
-      "rewards/margins": 0.22852472960948944,
-      "rewards/rejected": -0.4382990002632141,
+      "logits/chosen": -0.12167753279209137,
+      "logits/rejected": 0.5305444002151489,
+      "logps/chosen": -298.165283203125,
+      "logps/rejected": -341.3959045410156,
+      "loss": 0.1206,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3022175431251526,
+      "rewards/margins": 0.4555422365665436,
+      "rewards/rejected": -0.7577598094940186,
       "step": 230
     },
     {
       "epoch": 0.5504587155963303,
-      "grad_norm": 3.200697411595364,
+      "grad_norm": 9.439667913770359,
       "learning_rate": 2.5e-07,
-      "logits/chosen": -1.6722291707992554,
-      "logits/rejected": -1.496265172958374,
-      "logps/chosen": -304.0669250488281,
-      "logps/rejected": -301.3495788574219,
-      "loss": 0.0352,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.16535088419914246,
-      "rewards/margins": 0.23464509844779968,
-      "rewards/rejected": -0.39999598264694214,
+      "logits/chosen": -0.21128520369529724,
+      "logits/rejected": 0.5969688296318054,
+      "logps/chosen": -321.5914306640625,
+      "logps/rejected": -339.0221862792969,
+      "loss": 0.1218,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.34059566259384155,
+      "rewards/margins": 0.43612655997276306,
+      "rewards/rejected": -0.7767221927642822,
       "step": 240
     },
     {
       "epoch": 0.573394495412844,
-      "grad_norm": 3.068087094061713,
+      "grad_norm": 9.292342727311365,
       "learning_rate": 2.2998577335247667e-07,
-      "logits/chosen": -1.8118059635162354,
-      "logits/rejected": -1.58005952835083,
-      "logps/chosen": -316.8459777832031,
-      "logps/rejected": -301.0585632324219,
-      "loss": 0.0346,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.1305214762687683,
-      "rewards/margins": 0.27562573552131653,
-      "rewards/rejected": -0.40614718198776245,
+      "logits/chosen": -0.020272482186555862,
+      "logits/rejected": 0.808585524559021,
+      "logps/chosen": -337.55767822265625,
+      "logps/rejected": -346.179443359375,
+      "loss": 0.1185,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.337638795375824,
+      "rewards/margins": 0.5197167992591858,
+      "rewards/rejected": -0.8573554754257202,
       "step": 250
     },
     {
       "epoch": 0.573394495412844,
-      "eval_logits/chosen": -1.719161868095398,
-      "eval_logits/rejected": -1.4870593547821045,
-      "eval_logps/chosen": -298.28277587890625,
-      "eval_logps/rejected": -291.9156494140625,
-      "eval_loss": 0.033715467900037766,
+      "eval_logits/chosen": -0.1745007336139679,
+      "eval_logits/rejected": 0.8515735268592834,
+      "eval_logps/chosen": -315.9549560546875,
+      "eval_logps/rejected": -335.7632751464844,
+      "eval_loss": 0.11658623069524765,
       "eval_rewards/accuracies": 0.7543103694915771,
-      "eval_rewards/chosen": -0.1319267898797989,
-      "eval_rewards/margins": 0.3220053017139435,
-      "eval_rewards/rejected": -0.4539320766925812,
-      "eval_runtime": 94.8489,
-      "eval_samples_per_second": 19.167,
-      "eval_steps_per_second": 0.306,
+      "eval_rewards/chosen": -0.3086487948894501,
+      "eval_rewards/margins": 0.5837592482566833,
+      "eval_rewards/rejected": -0.8924079537391663,
+      "eval_runtime": 92.3611,
+      "eval_samples_per_second": 19.684,
+      "eval_steps_per_second": 0.314,
       "step": 250
     },
     {
       "epoch": 0.5963302752293578,
-      "grad_norm": 2.973513185196363,
+      "grad_norm": 8.186794570934321,
       "learning_rate": 2.1010002624165524e-07,
-      "logits/chosen": -1.625998854637146,
-      "logits/rejected": -1.528102159500122,
-      "logps/chosen": -289.2758483886719,
-      "logps/rejected": -324.4759216308594,
-      "loss": 0.0343,
+      "logits/chosen": -0.006458556745201349,
+      "logits/rejected": 0.6227847933769226,
+      "logps/chosen": -308.7506408691406,
+      "logps/rejected": -372.9380798339844,
+      "loss": 0.1168,
       "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.18561328947544098,
-      "rewards/margins": 0.3152090609073639,
-      "rewards/rejected": -0.5008223652839661,
+      "rewards/chosen": -0.3803611397743225,
+      "rewards/margins": 0.6050828695297241,
+      "rewards/rejected": -0.9854438900947571,
       "step": 260
     },
     {
       "epoch": 0.6192660550458715,
-      "grad_norm": 3.3992650920748635,
+      "grad_norm": 10.337683797583873,
       "learning_rate": 1.9047041344135043e-07,
-      "logits/chosen": -1.468390703201294,
-      "logits/rejected": -1.4237253665924072,
-      "logps/chosen": -287.03558349609375,
-      "logps/rejected": -301.84613037109375,
-      "loss": 0.0339,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.209732323884964,
-      "rewards/margins": 0.2902582287788391,
-      "rewards/rejected": -0.4999905526638031,
+      "logits/chosen": 0.44111576676368713,
+      "logits/rejected": 0.8292443156242371,
+      "logps/chosen": -313.4148254394531,
+      "logps/rejected": -354.33258056640625,
+      "loss": 0.1168,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.47352510690689087,
+      "rewards/margins": 0.5513299703598022,
+      "rewards/rejected": -1.0248548984527588,
       "step": 270
     },
     {
       "epoch": 0.6422018348623854,
-      "grad_norm": 3.2930402827157863,
+      "grad_norm": 10.116957119119249,
       "learning_rate": 1.7122294549409482e-07,
-      "logits/chosen": -1.6038357019424438,
-      "logits/rejected": -1.4040790796279907,
-      "logps/chosen": -287.1599426269531,
-      "logps/rejected": -311.6736755371094,
-      "loss": 0.0349,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.14161059260368347,
-      "rewards/margins": 0.3371681869029999,
-      "rewards/rejected": -0.4787788391113281,
+      "logits/chosen": 0.1775570809841156,
+      "logits/rejected": 1.0465366840362549,
+      "logps/chosen": -317.04180908203125,
+      "logps/rejected": -370.72479248046875,
+      "loss": 0.1221,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.4404297471046448,
+      "rewards/margins": 0.6288600564002991,
+      "rewards/rejected": -1.0692898035049438,
       "step": 280
     },
     {
       "epoch": 0.6651376146788991,
-      "grad_norm": 3.539437987763887,
+      "grad_norm": 9.609746165977107,
       "learning_rate": 1.524811797977383e-07,
-      "logits/chosen": -1.6025069952011108,
-      "logits/rejected": -1.4434142112731934,
-      "logps/chosen": -300.49578857421875,
-      "logps/rejected": -295.3720397949219,
-      "loss": 0.0334,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.12764978408813477,
-      "rewards/margins": 0.2869029641151428,
-      "rewards/rejected": -0.4145527780056,
+      "logits/chosen": 0.02145857736468315,
+      "logits/rejected": 0.823032021522522,
+      "logps/chosen": -322.24542236328125,
+      "logps/rejected": -342.22503662109375,
+      "loss": 0.1156,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.3451458811759949,
+      "rewards/margins": 0.5379365086555481,
+      "rewards/rejected": -0.883082389831543,
       "step": 290
     },
     {
       "epoch": 0.6880733944954128,
-      "grad_norm": 3.11392798700287,
+      "grad_norm": 9.08082569283608,
       "learning_rate": 1.3436542743979125e-07,
-      "logits/chosen": -1.6770799160003662,
-      "logits/rejected": -1.6160328388214111,
-      "logps/chosen": -319.42236328125,
-      "logps/rejected": -295.0827331542969,
-      "loss": 0.036,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.1347302943468094,
-      "rewards/margins": 0.22749857604503632,
-      "rewards/rejected": -0.3622289299964905,
+      "logits/chosen": -0.18167677521705627,
+      "logits/rejected": 0.25138911604881287,
+      "logps/chosen": -341.6298828125,
+      "logps/rejected": -338.09869384765625,
+      "loss": 0.1228,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.3568056523799896,
+      "rewards/margins": 0.43558257818222046,
+      "rewards/rejected": -0.7923881411552429,
       "step": 300
     },
     {
       "epoch": 0.6880733944954128,
-      "eval_logits/chosen": -1.6834876537322998,
-      "eval_logits/rejected": -1.4841814041137695,
-      "eval_logps/chosen": -298.45037841796875,
-      "eval_logps/rejected": -289.4286193847656,
-      "eval_loss": 0.03310078755021095,
-      "eval_rewards/accuracies": 0.75,
-      "eval_rewards/chosen": -0.13360273838043213,
-      "eval_rewards/margins": 0.29545897245407104,
-      "eval_rewards/rejected": -0.4290617108345032,
-      "eval_runtime": 94.9394,
-      "eval_samples_per_second": 19.149,
-      "eval_steps_per_second": 0.305,
+      "eval_logits/chosen": -0.13157324492931366,
+      "eval_logits/rejected": 0.8573818206787109,
+      "eval_logps/chosen": -322.0434265136719,
+      "eval_logps/rejected": -339.1875305175781,
+      "eval_loss": 0.11548212170600891,
+      "eval_rewards/accuracies": 0.7456896305084229,
+      "eval_rewards/chosen": -0.3695334494113922,
+      "eval_rewards/margins": 0.5571174621582031,
+      "eval_rewards/rejected": -0.9266510009765625,
+      "eval_runtime": 91.8257,
+      "eval_samples_per_second": 19.798,
+      "eval_steps_per_second": 0.316,
       "step": 300
     },
     {
       "epoch": 0.7110091743119266,
-      "grad_norm": 5.19818595642257,
+      "grad_norm": 9.32642322141669,
       "learning_rate": 1.1699198087116588e-07,
-      "logits/chosen": -1.741039514541626,
-      "logits/rejected": -1.6022865772247314,
-      "logps/chosen": -294.25433349609375,
-      "logps/rejected": -306.18817138671875,
-      "loss": 0.0357,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.16289182007312775,
-      "rewards/margins": 0.23449257016181946,
-      "rewards/rejected": -0.397384375333786,
+      "logits/chosen": -0.07652176916599274,
+      "logits/rejected": 0.6259859800338745,
+      "logps/chosen": -321.20538330078125,
+      "logps/rejected": -355.15618896484375,
+      "loss": 0.1239,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.43240228295326233,
+      "rewards/margins": 0.4546627104282379,
+      "rewards/rejected": -0.887065052986145,
       "step": 310
     },
     {
       "epoch": 0.7339449541284404,
-      "grad_norm": 3.4771448226315957,
+      "grad_norm": 9.870138396091912,
       "learning_rate": 1.00472367377196e-07,
-      "logits/chosen": -1.6833099126815796,
-      "logits/rejected": -1.5138328075408936,
-      "logps/chosen": -290.2877197265625,
-      "logps/rejected": -281.9583435058594,
-      "loss": 0.0353,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.16055694222450256,
-      "rewards/margins": 0.3141244947910309,
-      "rewards/rejected": -0.47468137741088867,
+      "logits/chosen": 0.15437743067741394,
+      "logits/rejected": 0.9939621686935425,
+      "logps/chosen": -314.3326416015625,
+      "logps/rejected": -335.50787353515625,
+      "loss": 0.1193,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.40100646018981934,
+      "rewards/margins": 0.6091704964637756,
+      "rewards/rejected": -1.0101768970489502,
       "step": 320
     },
     {
       "epoch": 0.7568807339449541,
-      "grad_norm": 3.5214688030455066,
+      "grad_norm": 10.470680354911353,
       "learning_rate": 8.49126331382102e-08,
-      "logits/chosen": -1.6057231426239014,
-      "logits/rejected": -1.4786289930343628,
-      "logps/chosen": -291.7789306640625,
-      "logps/rejected": -288.677001953125,
-      "loss": 0.0348,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.17256753146648407,
-      "rewards/margins": 0.24320188164710999,
-      "rewards/rejected": -0.41576939821243286,
+      "logits/chosen": 0.41432857513427734,
+      "logits/rejected": 0.9864773750305176,
+      "logps/chosen": -319.98114013671875,
+      "logps/rejected": -336.5364074707031,
+      "loss": 0.1183,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.45458975434303284,
+      "rewards/margins": 0.43977364897727966,
+      "rewards/rejected": -0.8943634033203125,
       "step": 330
     },
     {
       "epoch": 0.7798165137614679,
-      "grad_norm": 3.5006792226854495,
+      "grad_norm": 10.41854471188631,
       "learning_rate": 7.041266247556812e-08,
-      "logits/chosen": -1.7440093755722046,
-      "logits/rejected": -1.6453059911727905,
-      "logps/chosen": -304.0636901855469,
-      "logps/rejected": -299.057861328125,
-      "loss": 0.0337,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.16071084141731262,
-      "rewards/margins": 0.23002786934375763,
-      "rewards/rejected": -0.39073869585990906,
+      "logits/chosen": 0.3841426968574524,
+      "logits/rejected": 0.9351291656494141,
+      "logps/chosen": -334.6560974121094,
+      "logps/rejected": -349.6623229980469,
+      "loss": 0.1152,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.4666348397731781,
+      "rewards/margins": 0.4301484227180481,
+      "rewards/rejected": -0.8967832326889038,
       "step": 340
     },
     {
       "epoch": 0.8027522935779816,
-      "grad_norm": 3.5127255839066094,
+      "grad_norm": 9.996573814918259,
       "learning_rate": 5.706553665319955e-08,
-      "logits/chosen": -1.7355537414550781,
-      "logits/rejected": -1.5236848592758179,
-      "logps/chosen": -292.27850341796875,
-      "logps/rejected": -280.9353332519531,
-      "loss": 0.0359,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.17787429690361023,
-      "rewards/margins": 0.25486475229263306,
-      "rewards/rejected": -0.4327389597892761,
+      "logits/chosen": 0.6401292085647583,
+      "logits/rejected": 1.3996922969818115,
+      "logps/chosen": -324.6096496582031,
+      "logps/rejected": -337.38555908203125,
+      "loss": 0.1213,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.5011857151985168,
+      "rewards/margins": 0.4960555136203766,
+      "rewards/rejected": -0.997241199016571,
       "step": 350
     },
     {
       "epoch": 0.8027522935779816,
-      "eval_logits/chosen": -1.6786376237869263,
-      "eval_logits/rejected": -1.465800166130066,
-      "eval_logps/chosen": -298.86663818359375,
-      "eval_logps/rejected": -291.2398681640625,
-      "eval_loss": 0.0327293798327446,
-      "eval_rewards/accuracies": 0.7586206793785095,
-      "eval_rewards/chosen": -0.1377655267715454,
-      "eval_rewards/margins": 0.30940860509872437,
-      "eval_rewards/rejected": -0.44717416167259216,
-      "eval_runtime": 93.7918,
-      "eval_samples_per_second": 19.383,
-      "eval_steps_per_second": 0.309,
+      "eval_logits/chosen": 0.5151604413986206,
+      "eval_logits/rejected": 1.5740225315093994,
+      "eval_logps/chosen": -329.048583984375,
+      "eval_logps/rejected": -348.0972900390625,
+      "eval_loss": 0.113578200340271,
+      "eval_rewards/accuracies": 0.7629310488700867,
+      "eval_rewards/chosen": -0.4395850598812103,
+      "eval_rewards/margins": 0.5761635303497314,
+      "eval_rewards/rejected": -1.0157485008239746,
+      "eval_runtime": 92.2267,
+      "eval_samples_per_second": 19.712,
+      "eval_steps_per_second": 0.314,
       "step": 350
     },
     {
       "epoch": 0.8256880733944955,
-      "grad_norm": 2.7935239013392237,
+      "grad_norm": 8.23863269387286,
       "learning_rate": 4.4956936350761005e-08,
-      "logits/chosen": -1.6336042881011963,
-      "logits/rejected": -1.5328700542449951,
-      "logps/chosen": -265.51251220703125,
-      "logps/rejected": -297.154541015625,
-      "loss": 0.0328,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.1319248378276825,
-      "rewards/margins": 0.2542495131492615,
-      "rewards/rejected": -0.38617435097694397,
+      "logits/chosen": 0.49835261702537537,
+      "logits/rejected": 1.0901247262954712,
+      "logps/chosen": -292.41424560546875,
+      "logps/rejected": -345.85540771484375,
+      "loss": 0.1131,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.40094178915023804,
+      "rewards/margins": 0.4722411632537842,
+      "rewards/rejected": -0.8731829524040222,
       "step": 360
     },
     {
       "epoch": 0.8486238532110092,
-      "grad_norm": 3.203845878056458,
+      "grad_norm": 10.10305933177313,
       "learning_rate": 3.416459164418123e-08,
-      "logits/chosen": -1.746138572692871,
-      "logits/rejected": -1.5625007152557373,
-      "logps/chosen": -313.5556640625,
-      "logps/rejected": -302.9591369628906,
-      "loss": 0.0326,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.13971158862113953,
-      "rewards/margins": 0.2641337513923645,
-      "rewards/rejected": -0.4038453996181488,
+      "logits/chosen": 0.16921785473823547,
+      "logits/rejected": 0.9673768877983093,
+      "logps/chosen": -340.80059814453125,
+      "logps/rejected": -353.12274169921875,
+      "loss": 0.1129,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.41216129064559937,
+      "rewards/margins": 0.4933200776576996,
+      "rewards/rejected": -0.9054813385009766,
       "step": 370
     },
     {
       "epoch": 0.8715596330275229,
-      "grad_norm": 3.9723478814606876,
+      "grad_norm": 9.150940276647933,
       "learning_rate": 2.475778302439524e-08,
-      "logits/chosen": -1.700884461402893,
-      "logits/rejected": -1.4876903295516968,
-      "logps/chosen": -308.3489074707031,
-      "logps/rejected": -299.65850830078125,
-      "loss": 0.0342,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.11970899254083633,
-      "rewards/margins": 0.3003136217594147,
-      "rewards/rejected": -0.4200226366519928,
+      "logits/chosen": 0.3220874071121216,
+      "logits/rejected": 1.2661678791046143,
+      "logps/chosen": -332.2058410644531,
+      "logps/rejected": -351.96856689453125,
+      "loss": 0.1174,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.35827815532684326,
+      "rewards/margins": 0.5848454236984253,
+      "rewards/rejected": -0.9431236386299133,
       "step": 380
     },
     {
       "epoch": 0.8944954128440367,
-      "grad_norm": 3.433718178917216,
+      "grad_norm": 9.488885903438641,
       "learning_rate": 1.6796896657433805e-08,
-      "logits/chosen": -1.6044851541519165,
-      "logits/rejected": -1.4463411569595337,
-      "logps/chosen": -265.7919006347656,
-      "logps/rejected": -267.8514099121094,
-      "loss": 0.0356,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.13834409415721893,
-      "rewards/margins": 0.2471076250076294,
-      "rewards/rejected": -0.3854517340660095,
+      "logits/chosen": 0.6681944727897644,
+      "logits/rejected": 1.2935806512832642,
+      "logps/chosen": -292.4158630371094,
+      "logps/rejected": -315.7539978027344,
+      "loss": 0.123,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.40458402037620544,
+      "rewards/margins": 0.4598938524723053,
+      "rewards/rejected": -0.8644779324531555,
       "step": 390
     },
     {
       "epoch": 0.9174311926605505,
-      "grad_norm": 3.420165449234039,
+      "grad_norm": 9.000204295912624,
       "learning_rate": 1.0333036740834855e-08,
-      "logits/chosen": -1.5561296939849854,
-      "logits/rejected": -1.4712246656417847,
-      "logps/chosen": -236.0593719482422,
-      "logps/rejected": -266.1527404785156,
-      "loss": 0.0351,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.11499743163585663,
-      "rewards/margins": 0.2543741464614868,
-      "rewards/rejected": -0.36937159299850464,
+      "logits/chosen": 0.7832537889480591,
+      "logits/rejected": 1.2279746532440186,
+      "logps/chosen": -260.876708984375,
+      "logps/rejected": -314.25390625,
+      "loss": 0.12,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3631708323955536,
+      "rewards/margins": 0.4872121214866638,
+      "rewards/rejected": -0.8503829836845398,
       "step": 400
     },
     {
       "epoch": 0.9174311926605505,
-      "eval_logits/chosen": -1.6862857341766357,
-      "eval_logits/rejected": -1.476194977760315,
-      "eval_logps/chosen": -297.669189453125,
-      "eval_logps/rejected": -289.0508728027344,
-      "eval_loss": 0.0327373668551445,
+      "eval_logits/chosen": 0.40672120451927185,
+      "eval_logits/rejected": 1.4352502822875977,
+      "eval_logps/chosen": -324.5692443847656,
+      "eval_logps/rejected": -342.49981689453125,
+      "eval_loss": 0.11321888864040375,
       "eval_rewards/accuracies": 0.7543103694915771,
-      "eval_rewards/chosen": -0.12579074501991272,
-      "eval_rewards/margins": 0.2994934022426605,
-      "eval_rewards/rejected": -0.42528414726257324,
-      "eval_runtime": 95.429,
-      "eval_samples_per_second": 19.051,
-      "eval_steps_per_second": 0.304,
+      "eval_rewards/chosen": -0.39479103684425354,
+      "eval_rewards/margins": 0.5649827122688293,
+      "eval_rewards/rejected": -0.9597737789154053,
+      "eval_runtime": 92.1464,
+      "eval_samples_per_second": 19.729,
+      "eval_steps_per_second": 0.315,
       "step": 400
     },
     {
       "epoch": 0.9403669724770642,
-      "grad_norm": 3.5258604798563833,
+      "grad_norm": 12.797858678996972,
       "learning_rate": 5.4076974448211685e-09,
-      "logits/chosen": -1.5972687005996704,
-      "logits/rejected": -1.4251827001571655,
-      "logps/chosen": -284.83367919921875,
-      "logps/rejected": -281.9726257324219,
-      "loss": 0.0339,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.13881805539131165,
-      "rewards/margins": 0.30044105648994446,
-      "rewards/rejected": -0.4392591118812561,
+      "logits/chosen": 0.733371913433075,
+      "logits/rejected": 1.6556295156478882,
+      "logps/chosen": -313.32049560546875,
+      "logps/rejected": -336.0906677246094,
+      "loss": 0.1171,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.42368635535240173,
+      "rewards/margins": 0.5567531585693359,
+      "rewards/rejected": -0.9804395437240601,
       "step": 410
     },
     {
       "epoch": 0.963302752293578,
-      "grad_norm": 5.008109192532265,
+      "grad_norm": 11.330514295442143,
       "learning_rate": 2.052496544188487e-09,
-      "logits/chosen": -1.5873286724090576,
-      "logits/rejected": -1.4294344186782837,
-      "logps/chosen": -275.04388427734375,
-      "logps/rejected": -290.7890319824219,
-      "loss": 0.0341,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.1560489982366562,
-      "rewards/margins": 0.293067991733551,
-      "rewards/rejected": -0.4491170048713684,
+      "logits/chosen": 0.8590083122253418,
+      "logits/rejected": 1.5339264869689941,
+      "logps/chosen": -304.95831298828125,
+      "logps/rejected": -347.80169677734375,
+      "loss": 0.1153,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.4551934599876404,
+      "rewards/margins": 0.5640500783920288,
+      "rewards/rejected": -1.019243597984314,
       "step": 420
     },
     {
       "epoch": 0.9862385321100917,
-      "grad_norm": 3.385882880558537,
+      "grad_norm": 9.086504274831077,
       "learning_rate": 2.889724508297886e-10,
-      "logits/chosen": -1.6612422466278076,
-      "logits/rejected": -1.3801202774047852,
-      "logps/chosen": -316.4678039550781,
-      "logps/rejected": -282.8833923339844,
-      "loss": 0.0343,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.13292355835437775,
-      "rewards/margins": 0.25554487109184265,
-      "rewards/rejected": -0.3884683847427368,
+      "logits/chosen": 0.5266289710998535,
+      "logits/rejected": 1.5435163974761963,
+      "logps/chosen": -344.6377868652344,
+      "logps/rejected": -336.5421447753906,
+      "loss": 0.1158,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.41462403535842896,
+      "rewards/margins": 0.5104321837425232,
+      "rewards/rejected": -0.9250561594963074,
       "step": 430
     },
     {
       "epoch": 1.0,
       "step": 436,
       "total_flos": 0.0,
-      "train_loss": 0.0384325077500912,
-      "train_runtime": 11659.9634,
-      "train_samples_per_second": 4.782,
-      "train_steps_per_second": 0.037
+      "train_loss": 0.13218348616853767,
+      "train_runtime": 11343.5766,
+      "train_samples_per_second": 4.915,
+      "train_steps_per_second": 0.038
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eea8da85d8ab87e00473fab7d78c768e11b3f63592d5a8e409a7abccfbac4f11
+oid sha256:5b97c68988ee761051dc175f846465ecfe6423242be1592fe1880198a31410e6
 size 7608