Hugging Face's logo

sfulay
/
zephyr-7b-dpo-full-gpt_consistent-reward-scale-1-rpo

Safetensors
mistral
trl
dpo
alignment-handbook
Generated from Trainer
Model card Files Community
sfulay commited on
Commit
9b16a50
•
1 Parent(s): 4d79451

Model save

Browse files
Files changed (9) hide show
  1. README.md +17 -17
  2. all_results.json +15 -15
  3. eval_results.json +12 -12
  4. model-00001-of-00003.safetensors +1 -1
  5. model-00002-of-00003.safetensors +1 -1
  6. model-00003-of-00003.safetensors +1 -1
  7. train_results.json +3 -3
  8. trainer_state.json +526 -526
  9. training_args.bin +1 -1
README.md CHANGED
@@ -18,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
18
 
19
  This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
20
  It achieves the following results on the evaluation set:
21
- - Loss: 0.1126
22
- - Rewards/chosen: -0.4018
23
- - Rewards/rejected: -0.9642
24
- - Rewards/accuracies: 0.75
25
- - Rewards/margins: 0.5623
26
- - Logps/rejected: -342.9399
27
- - Logps/chosen: -325.2745
28
- - Logits/rejected: 0.1904
29
- - Logits/chosen: -0.6159
30
 
31
  ## Model description
32
 
@@ -63,14 +63,14 @@ The following hyperparameters were used during training:
63
 
64
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
65
  |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
66
- | 0.1732 | 0.1147 | 50 | 0.1640 | 0.0140 | -0.1117 | 0.7069 | 0.1256 | -257.6913 | -283.6945 | -2.4927 | -2.5723 |
67
- | 0.1412 | 0.2294 | 100 | 0.1392 | -0.5601 | -0.9388 | 0.7198 | 0.3787 | -340.4027 | -341.0970 | -0.8338 | -1.1029 |
68
- | 0.1327 | 0.3440 | 150 | 0.1257 | -0.3660 | -0.8686 | 0.7241 | 0.5025 | -333.3776 | -321.6920 | -0.4213 | -0.9346 |
69
- | 0.1242 | 0.4587 | 200 | 0.1193 | -0.3424 | -0.8506 | 0.7457 | 0.5082 | -331.5807 | -319.3252 | -0.1498 | -0.9084 |
70
- | 0.118 | 0.5734 | 250 | 0.1167 | -0.3734 | -0.9685 | 0.7543 | 0.5950 | -343.3683 | -322.4342 | -0.0251 | -0.8147 |
71
- | 0.1236 | 0.6881 | 300 | 0.1145 | -0.4780 | -1.0449 | 0.7629 | 0.5669 | -351.0162 | -332.8936 | 0.2891 | -0.5285 |
72
- | 0.122 | 0.8028 | 350 | 0.1127 | -0.4577 | -1.0376 | 0.7672 | 0.5799 | -350.2830 | -330.8570 | 0.3397 | -0.4806 |
73
- | 0.12 | 0.9174 | 400 | 0.1126 | -0.4018 | -0.9642 | 0.75 | 0.5623 | -342.9399 | -325.2745 | 0.1904 | -0.6159 |
74
 
75
 
76
  ### Framework versions
 
18
 
19
  This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
20
  It achieves the following results on the evaluation set:
21
+ - Loss: 0.0087
22
+ - Rewards/chosen: -0.0010
23
+ - Rewards/rejected: -0.1538
24
+ - Rewards/accuracies: 0.7759
25
+ - Rewards/margins: 0.1527
26
+ - Logps/rejected: -261.8980
27
+ - Logps/chosen: -285.1917
28
+ - Logits/rejected: -2.3893
29
+ - Logits/chosen: -2.4834
30
 
31
  ## Model description
32
 
 
63
 
64
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
65
  |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
66
+ | 0.012 | 0.1147 | 50 | 0.0113 | 0.0776 | -0.0073 | 0.6983 | 0.0849 | -247.2525 | -277.3343 | -2.5025 | -2.5786 |
67
+ | 0.0111 | 0.2294 | 100 | 0.0100 | 0.0400 | -0.0817 | 0.7112 | 0.1217 | -254.6882 | -281.0889 | -2.3452 | -2.4456 |
68
+ | 0.0104 | 0.3440 | 150 | 0.0098 | -0.0092 | -0.1421 | 0.7284 | 0.1329 | -260.7338 | -286.0115 | -2.4006 | -2.4971 |
69
+ | 0.0096 | 0.4587 | 200 | 0.0093 | 0.0230 | -0.1186 | 0.7888 | 0.1416 | -258.3851 | -282.7939 | -2.4206 | -2.5115 |
70
+ | 0.0093 | 0.5734 | 250 | 0.0089 | -0.0116 | -0.1682 | 0.7845 | 0.1565 | -263.3386 | -286.2548 | -2.3653 | -2.4591 |
71
+ | 0.0096 | 0.6881 | 300 | 0.0088 | -0.0083 | -0.1589 | 0.7845 | 0.1506 | -262.4115 | -285.9173 | -2.3891 | -2.4814 |
72
+ | 0.0096 | 0.8028 | 350 | 0.0087 | -0.0041 | -0.1596 | 0.7802 | 0.1555 | -262.4817 | -285.5014 | -2.3906 | -2.4846 |
73
+ | 0.0093 | 0.9174 | 400 | 0.0087 | -0.0010 | -0.1538 | 0.7759 | 0.1527 | -261.8980 | -285.1917 | -2.3893 | -2.4834 |
74
 
75
 
76
  ### Framework versions
all_results.json CHANGED
@@ -1,22 +1,22 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": 0.41779446601867676,
4
- "eval_logits/rejected": 1.454390525817871,
5
- "eval_logps/chosen": -324.46234130859375,
6
- "eval_logps/rejected": -342.5584716796875,
7
- "eval_loss": 0.11311393231153488,
8
- "eval_rewards/accuracies": 0.7543103694915771,
9
- "eval_rewards/chosen": -0.393722802400589,
10
- "eval_rewards/margins": 0.566637396812439,
11
- "eval_rewards/rejected": -0.9603601098060608,
12
- "eval_runtime": 93.576,
13
  "eval_samples": 1818,
14
- "eval_samples_per_second": 19.428,
15
- "eval_steps_per_second": 0.31,
16
  "total_flos": 0.0,
17
- "train_loss": 0.13197535372108493,
18
- "train_runtime": 11523.0815,
19
  "train_samples": 55758,
20
- "train_samples_per_second": 4.839,
21
  "train_steps_per_second": 0.038
22
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -0.6058345437049866,
4
+ "eval_logits/rejected": 0.2021634727716446,
5
+ "eval_logps/chosen": -325.1861572265625,
6
+ "eval_logps/rejected": -342.98297119140625,
7
+ "eval_loss": 0.11247362196445465,
8
+ "eval_rewards/accuracies": 0.75,
9
+ "eval_rewards/chosen": -0.4009607136249542,
10
+ "eval_rewards/margins": 0.5636443495750427,
11
+ "eval_rewards/rejected": -0.9646049737930298,
12
+ "eval_runtime": 95.1407,
13
  "eval_samples": 1818,
14
+ "eval_samples_per_second": 19.109,
15
+ "eval_steps_per_second": 0.305,
16
  "total_flos": 0.0,
17
+ "train_loss": 0.010263856175705927,
18
+ "train_runtime": 11602.0724,
19
  "train_samples": 55758,
20
+ "train_samples_per_second": 4.806,
21
  "train_steps_per_second": 0.038
22
  }
eval_results.json CHANGED
@@ -1,16 +1,16 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": 0.41779446601867676,
4
- "eval_logits/rejected": 1.454390525817871,
5
- "eval_logps/chosen": -324.46234130859375,
6
- "eval_logps/rejected": -342.5584716796875,
7
- "eval_loss": 0.11311393231153488,
8
- "eval_rewards/accuracies": 0.7543103694915771,
9
- "eval_rewards/chosen": -0.393722802400589,
10
- "eval_rewards/margins": 0.566637396812439,
11
- "eval_rewards/rejected": -0.9603601098060608,
12
- "eval_runtime": 93.576,
13
  "eval_samples": 1818,
14
- "eval_samples_per_second": 19.428,
15
- "eval_steps_per_second": 0.31
16
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -0.6058345437049866,
4
+ "eval_logits/rejected": 0.2021634727716446,
5
+ "eval_logps/chosen": -325.1861572265625,
6
+ "eval_logps/rejected": -342.98297119140625,
7
+ "eval_loss": 0.11247362196445465,
8
+ "eval_rewards/accuracies": 0.75,
9
+ "eval_rewards/chosen": -0.4009607136249542,
10
+ "eval_rewards/margins": 0.5636443495750427,
11
+ "eval_rewards/rejected": -0.9646049737930298,
12
+ "eval_runtime": 95.1407,
13
  "eval_samples": 1818,
14
+ "eval_samples_per_second": 19.109,
15
+ "eval_steps_per_second": 0.305
16
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1e0e9f49e5125e802deca218e3a20f75ae9597d6c0bfe6256478d7a39e082ac0
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e83a491be5a488dd60f70f10e7c0d05d20b183d2ef618c8cf88af99fd0fbda9
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7e9cabac2e92bfb43293f329c88741c80f6d4c2811430c129984ce94af6a917d
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c51d682460a19433562ccf0c67611e482107132b105270ad5f1469253d17112a
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:50e47b05e3ffc16c276de51ad276bb767d2f2baf1107bb39525ad6872def7e26
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d5ab96bc5580ea9868437915d82fb29baa178e821404e23189c9fe86e831bb30
3
  size 4540516344
train_results.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "epoch": 1.0,
3
  "total_flos": 0.0,
4
- "train_loss": 0.13197535372108493,
5
- "train_runtime": 11523.0815,
6
  "train_samples": 55758,
7
- "train_samples_per_second": 4.839,
8
  "train_steps_per_second": 0.038
9
  }
 
1
  {
2
  "epoch": 1.0,
3
  "total_flos": 0.0,
4
+ "train_loss": 0.010263856175705927,
5
+ "train_runtime": 11602.0724,
6
  "train_samples": 55758,
7
+ "train_samples_per_second": 4.806,
8
  "train_steps_per_second": 0.038
9
  }
trainer_state.json CHANGED
@@ -10,784 +10,784 @@
10
  "log_history": [
11
  {
12
  "epoch": 0.022935779816513763,
13
- "grad_norm": 5.604829014017037,
14
  "learning_rate": 1.1363636363636363e-07,
15
- "logits/chosen": -2.619293689727783,
16
- "logits/rejected": -2.5524563789367676,
17
- "logps/chosen": -265.4638977050781,
18
- "logps/rejected": -236.136962890625,
19
- "loss": 0.1942,
20
- "rewards/accuracies": 0.4000000059604645,
21
- "rewards/chosen": -0.00018078314315062016,
22
- "rewards/margins": -0.00011108322360087186,
23
- "rewards/rejected": -6.969989772187546e-05,
24
  "step": 10
25
  },
26
  {
27
  "epoch": 0.045871559633027525,
28
- "grad_norm": 5.250583616242791,
29
  "learning_rate": 2.2727272727272726e-07,
30
- "logits/chosen": -2.6577353477478027,
31
- "logits/rejected": -2.575864315032959,
32
- "logps/chosen": -298.8517150878906,
33
- "logps/rejected": -274.293212890625,
34
- "loss": 0.1959,
35
- "rewards/accuracies": 0.6312500238418579,
36
- "rewards/chosen": -0.0006643417873419821,
37
- "rewards/margins": 0.0011681303149089217,
38
- "rewards/rejected": -0.0018324721604585648,
39
  "step": 20
40
  },
41
  {
42
  "epoch": 0.06880733944954129,
43
- "grad_norm": 4.980077918182304,
44
  "learning_rate": 3.4090909090909085e-07,
45
- "logits/chosen": -2.675644636154175,
46
- "logits/rejected": -2.6021487712860107,
47
- "logps/chosen": -290.4170227050781,
48
- "logps/rejected": -234.3779296875,
49
- "loss": 0.1923,
50
- "rewards/accuracies": 0.675000011920929,
51
- "rewards/chosen": 0.009157607331871986,
52
- "rewards/margins": 0.013118197210133076,
53
- "rewards/rejected": -0.003960589878261089,
54
  "step": 30
55
  },
56
  {
57
  "epoch": 0.09174311926605505,
58
- "grad_norm": 5.215480488397355,
59
  "learning_rate": 4.545454545454545e-07,
60
- "logits/chosen": -2.660599946975708,
61
- "logits/rejected": -2.6108055114746094,
62
- "logps/chosen": -281.02667236328125,
63
- "logps/rejected": -267.7845764160156,
64
- "loss": 0.1835,
65
- "rewards/accuracies": 0.675000011920929,
66
- "rewards/chosen": 0.04142485558986664,
67
- "rewards/margins": 0.04134037345647812,
68
- "rewards/rejected": 8.447784784948453e-05,
69
  "step": 40
70
  },
71
  {
72
  "epoch": 0.11467889908256881,
73
- "grad_norm": 5.6967811731593905,
74
  "learning_rate": 4.997110275491701e-07,
75
- "logits/chosen": -2.619861602783203,
76
- "logits/rejected": -2.6124091148376465,
77
- "logps/chosen": -293.50762939453125,
78
- "logps/rejected": -303.9714660644531,
79
- "loss": 0.1732,
80
- "rewards/accuracies": 0.6812499761581421,
81
- "rewards/chosen": 0.027435380965471268,
82
- "rewards/margins": 0.0753081664443016,
83
- "rewards/rejected": -0.04787278175354004,
84
  "step": 50
85
  },
86
  {
87
  "epoch": 0.11467889908256881,
88
- "eval_logits/chosen": -2.5723471641540527,
89
- "eval_logits/rejected": -2.49267315864563,
90
- "eval_logps/chosen": -283.6944580078125,
91
- "eval_logps/rejected": -257.6913146972656,
92
- "eval_loss": 0.1639617681503296,
93
- "eval_rewards/accuracies": 0.7068965435028076,
94
- "eval_rewards/chosen": 0.013956375420093536,
95
- "eval_rewards/margins": 0.12564486265182495,
96
- "eval_rewards/rejected": -0.11168847978115082,
97
- "eval_runtime": 95.9923,
98
- "eval_samples_per_second": 18.939,
99
- "eval_steps_per_second": 0.302,
100
  "step": 50
101
  },
102
  {
103
  "epoch": 0.13761467889908258,
104
- "grad_norm": 7.248589740263705,
105
  "learning_rate": 4.979475034558115e-07,
106
- "logits/chosen": -2.5621228218078613,
107
- "logits/rejected": -2.503438949584961,
108
- "logps/chosen": -293.131103515625,
109
- "logps/rejected": -272.94793701171875,
110
- "loss": 0.1592,
111
- "rewards/accuracies": 0.699999988079071,
112
- "rewards/chosen": -0.03646404668688774,
113
- "rewards/margins": 0.11744984239339828,
114
- "rewards/rejected": -0.15391388535499573,
115
  "step": 60
116
  },
117
  {
118
  "epoch": 0.16055045871559634,
119
- "grad_norm": 7.653772817525038,
120
  "learning_rate": 4.945923025551788e-07,
121
- "logits/chosen": -2.4572176933288574,
122
- "logits/rejected": -2.407254219055176,
123
- "logps/chosen": -327.6614685058594,
124
- "logps/rejected": -284.7535705566406,
125
- "loss": 0.1557,
126
- "rewards/accuracies": 0.6937500238418579,
127
- "rewards/chosen": -0.0009046800551004708,
128
- "rewards/margins": 0.23255476355552673,
129
- "rewards/rejected": -0.23345942795276642,
130
  "step": 70
131
  },
132
  {
133
  "epoch": 0.1834862385321101,
134
- "grad_norm": 8.127224192262021,
135
  "learning_rate": 4.896669632591651e-07,
136
- "logits/chosen": -2.3246560096740723,
137
- "logits/rejected": -2.2035281658172607,
138
- "logps/chosen": -297.11651611328125,
139
- "logps/rejected": -290.16302490234375,
140
- "loss": 0.1475,
141
- "rewards/accuracies": 0.7437499761581421,
142
- "rewards/chosen": -0.11266676336526871,
143
- "rewards/margins": 0.2624030113220215,
144
- "rewards/rejected": -0.3750697076320648,
145
  "step": 80
146
  },
147
  {
148
  "epoch": 0.20642201834862386,
149
- "grad_norm": 8.777001707426516,
150
  "learning_rate": 4.832031033425662e-07,
151
- "logits/chosen": -1.9738223552703857,
152
- "logits/rejected": -1.9115241765975952,
153
- "logps/chosen": -299.46820068359375,
154
- "logps/rejected": -291.00994873046875,
155
- "loss": 0.1449,
156
- "rewards/accuracies": 0.71875,
157
- "rewards/chosen": -0.11874761432409286,
158
- "rewards/margins": 0.3233392536640167,
159
- "rewards/rejected": -0.4420868456363678,
160
  "step": 90
161
  },
162
  {
163
  "epoch": 0.22935779816513763,
164
- "grad_norm": 9.678897311244762,
165
  "learning_rate": 4.752422169756047e-07,
166
- "logits/chosen": -1.0515625476837158,
167
- "logits/rejected": -1.0063254833221436,
168
- "logps/chosen": -311.99945068359375,
169
- "logps/rejected": -342.3149719238281,
170
- "loss": 0.1412,
171
- "rewards/accuracies": 0.6875,
172
- "rewards/chosen": -0.39574748277664185,
173
- "rewards/margins": 0.3081180453300476,
174
- "rewards/rejected": -0.7038655281066895,
175
  "step": 100
176
  },
177
  {
178
  "epoch": 0.22935779816513763,
179
- "eval_logits/chosen": -1.1028852462768555,
180
- "eval_logits/rejected": -0.8337642550468445,
181
- "eval_logps/chosen": -341.0970458984375,
182
- "eval_logps/rejected": -340.4026794433594,
183
- "eval_loss": 0.13921599090099335,
184
- "eval_rewards/accuracies": 0.7198275923728943,
185
- "eval_rewards/chosen": -0.5600696802139282,
186
- "eval_rewards/margins": 0.3787323534488678,
187
- "eval_rewards/rejected": -0.9388020038604736,
188
- "eval_runtime": 95.7518,
189
- "eval_samples_per_second": 18.987,
190
- "eval_steps_per_second": 0.303,
191
  "step": 100
192
  },
193
  {
194
  "epoch": 0.25229357798165136,
195
- "grad_norm": 8.075119669591446,
196
  "learning_rate": 4.658354083558188e-07,
197
- "logits/chosen": -1.309912085533142,
198
- "logits/rejected": -1.0928936004638672,
199
- "logps/chosen": -323.1109313964844,
200
- "logps/rejected": -320.5851135253906,
201
- "loss": 0.1371,
202
- "rewards/accuracies": 0.71875,
203
- "rewards/chosen": -0.5792280435562134,
204
- "rewards/margins": 0.2892912030220032,
205
- "rewards/rejected": -0.868519127368927,
206
  "step": 110
207
  },
208
  {
209
  "epoch": 0.27522935779816515,
210
- "grad_norm": 11.527192184915142,
211
  "learning_rate": 4.550430636492389e-07,
212
- "logits/chosen": -1.2554824352264404,
213
- "logits/rejected": -1.2111155986785889,
214
- "logps/chosen": -315.65582275390625,
215
- "logps/rejected": -323.9941101074219,
216
- "loss": 0.1347,
217
- "rewards/accuracies": 0.6812499761581421,
218
- "rewards/chosen": -0.39836472272872925,
219
- "rewards/margins": 0.3269372582435608,
220
- "rewards/rejected": -0.7253020405769348,
221
  "step": 120
222
  },
223
  {
224
  "epoch": 0.2981651376146789,
225
- "grad_norm": 10.253713539855786,
226
  "learning_rate": 4.429344633468004e-07,
227
- "logits/chosen": -0.9448979496955872,
228
- "logits/rejected": -0.6800575256347656,
229
- "logps/chosen": -294.2066345214844,
230
- "logps/rejected": -319.7179870605469,
231
- "loss": 0.1367,
232
  "rewards/accuracies": 0.699999988079071,
233
- "rewards/chosen": -0.38026565313339233,
234
- "rewards/margins": 0.39078813791275024,
235
- "rewards/rejected": -0.7710537910461426,
236
  "step": 130
237
  },
238
  {
239
  "epoch": 0.3211009174311927,
240
- "grad_norm": 11.166916308293251,
241
  "learning_rate": 4.2958733752443187e-07,
242
- "logits/chosen": -0.8638193011283875,
243
- "logits/rejected": -0.5810004472732544,
244
- "logps/chosen": -307.26165771484375,
245
- "logps/rejected": -287.42547607421875,
246
- "loss": 0.134,
247
- "rewards/accuracies": 0.706250011920929,
248
- "rewards/chosen": -0.3107650578022003,
249
- "rewards/margins": 0.3205246925354004,
250
- "rewards/rejected": -0.6312897801399231,
251
  "step": 140
252
  },
253
  {
254
  "epoch": 0.3440366972477064,
255
- "grad_norm": 10.491465778765697,
256
  "learning_rate": 4.150873668617898e-07,
257
- "logits/chosen": -0.785676121711731,
258
- "logits/rejected": -0.4156356453895569,
259
- "logps/chosen": -306.7540588378906,
260
- "logps/rejected": -312.4991149902344,
261
- "loss": 0.1327,
262
- "rewards/accuracies": 0.71875,
263
- "rewards/chosen": -0.40210476517677307,
264
- "rewards/margins": 0.39101067185401917,
265
- "rewards/rejected": -0.793115496635437,
266
  "step": 150
267
  },
268
  {
269
  "epoch": 0.3440366972477064,
270
- "eval_logits/chosen": -0.934603214263916,
271
- "eval_logits/rejected": -0.4213140308856964,
272
- "eval_logps/chosen": -321.6920166015625,
273
- "eval_logps/rejected": -333.3775939941406,
274
- "eval_loss": 0.1256996989250183,
275
- "eval_rewards/accuracies": 0.7241379022598267,
276
- "eval_rewards/chosen": -0.36601918935775757,
277
- "eval_rewards/margins": 0.5025321841239929,
278
- "eval_rewards/rejected": -0.8685513734817505,
279
- "eval_runtime": 95.7427,
280
- "eval_samples_per_second": 18.988,
281
- "eval_steps_per_second": 0.303,
282
  "step": 150
283
  },
284
  {
285
  "epoch": 0.3669724770642202,
286
- "grad_norm": 10.606572226202415,
287
  "learning_rate": 3.9952763262280397e-07,
288
- "logits/chosen": -0.6937485933303833,
289
- "logits/rejected": -0.39156150817871094,
290
- "logps/chosen": -341.2958068847656,
291
- "logps/rejected": -376.64788818359375,
292
- "loss": 0.1226,
293
- "rewards/accuracies": 0.737500011920929,
294
- "rewards/chosen": -0.4474472105503082,
295
- "rewards/margins": 0.4593452513217926,
296
- "rewards/rejected": -0.9067924618721008,
297
  "step": 160
298
  },
299
  {
300
  "epoch": 0.38990825688073394,
301
- "grad_norm": 7.909933445936232,
302
  "learning_rate": 3.8300801912883414e-07,
303
- "logits/chosen": -0.5848881602287292,
304
- "logits/rejected": -0.295288622379303,
305
- "logps/chosen": -314.4745178222656,
306
- "logps/rejected": -355.83209228515625,
307
- "loss": 0.1194,
308
- "rewards/accuracies": 0.699999988079071,
309
- "rewards/chosen": -0.42873620986938477,
310
- "rewards/margins": 0.4288958013057709,
311
- "rewards/rejected": -0.857632040977478,
312
  "step": 170
313
  },
314
  {
315
  "epoch": 0.41284403669724773,
316
- "grad_norm": 9.296204512324422,
317
  "learning_rate": 3.6563457256020884e-07,
318
- "logits/chosen": -0.4405382573604584,
319
- "logits/rejected": 0.17199954390525818,
320
- "logps/chosen": -351.5772399902344,
321
- "logps/rejected": -332.4178771972656,
322
- "loss": 0.1256,
323
- "rewards/accuracies": 0.7250000238418579,
324
- "rewards/chosen": -0.48219355940818787,
325
- "rewards/margins": 0.4570356011390686,
326
- "rewards/rejected": -0.9392291903495789,
327
  "step": 180
328
  },
329
  {
330
  "epoch": 0.43577981651376146,
331
- "grad_norm": 9.788859469537575,
332
  "learning_rate": 3.475188202022617e-07,
333
- "logits/chosen": -0.9616500735282898,
334
- "logits/rejected": -0.5780771970748901,
335
- "logps/chosen": -284.89788818359375,
336
- "logps/rejected": -339.87139892578125,
337
- "loss": 0.124,
338
- "rewards/accuracies": 0.7749999761581421,
339
- "rewards/chosen": -0.2678338885307312,
340
- "rewards/margins": 0.487435519695282,
341
- "rewards/rejected": -0.7552694082260132,
342
  "step": 190
343
  },
344
  {
345
  "epoch": 0.45871559633027525,
346
- "grad_norm": 8.31792118291438,
347
  "learning_rate": 3.287770545059052e-07,
348
- "logits/chosen": -0.9191502332687378,
349
- "logits/rejected": -0.3312894105911255,
350
- "logps/chosen": -313.3896179199219,
351
- "logps/rejected": -324.57000732421875,
352
- "loss": 0.1242,
353
- "rewards/accuracies": 0.768750011920929,
354
- "rewards/chosen": -0.3510800302028656,
355
- "rewards/margins": 0.4560603201389313,
356
- "rewards/rejected": -0.8071403503417969,
357
  "step": 200
358
  },
359
  {
360
  "epoch": 0.45871559633027525,
361
- "eval_logits/chosen": -0.9084411859512329,
362
- "eval_logits/rejected": -0.1497526615858078,
363
- "eval_logps/chosen": -319.3252258300781,
364
- "eval_logps/rejected": -331.5806884765625,
365
- "eval_loss": 0.11925708502531052,
366
- "eval_rewards/accuracies": 0.7456896305084229,
367
- "eval_rewards/chosen": -0.34235113859176636,
368
- "eval_rewards/margins": 0.5082312226295471,
369
- "eval_rewards/rejected": -0.8505823612213135,
370
- "eval_runtime": 93.7494,
371
- "eval_samples_per_second": 19.392,
372
- "eval_steps_per_second": 0.309,
373
  "step": 200
374
  },
375
  {
376
  "epoch": 0.481651376146789,
377
- "grad_norm": 7.584339731807936,
378
  "learning_rate": 3.0952958655864954e-07,
379
- "logits/chosen": -0.6695399284362793,
380
- "logits/rejected": -0.2294241189956665,
381
- "logps/chosen": -311.20343017578125,
382
- "logps/rejected": -338.2936096191406,
383
- "loss": 0.1173,
384
- "rewards/accuracies": 0.762499988079071,
385
- "rewards/chosen": -0.3789902925491333,
386
- "rewards/margins": 0.4605233073234558,
387
- "rewards/rejected": -0.8395135998725891,
388
  "step": 210
389
  },
390
  {
391
  "epoch": 0.5045871559633027,
392
- "grad_norm": 10.1245037946263,
393
  "learning_rate": 2.898999737583448e-07,
394
- "logits/chosen": -0.6778780221939087,
395
- "logits/rejected": 0.15124213695526123,
396
- "logps/chosen": -365.30010986328125,
397
- "logps/rejected": -385.3759460449219,
398
- "loss": 0.1146,
399
- "rewards/accuracies": 0.793749988079071,
400
- "rewards/chosen": -0.42619770765304565,
401
- "rewards/margins": 0.5778595209121704,
402
- "rewards/rejected": -1.0040571689605713,
403
  "step": 220
404
  },
405
  {
406
  "epoch": 0.5275229357798165,
407
- "grad_norm": 8.611834221562093,
408
  "learning_rate": 2.7001422664752333e-07,
409
- "logits/chosen": -0.5495241284370422,
410
- "logits/rejected": -0.10778895765542984,
411
- "logps/chosen": -308.42864990234375,
412
- "logps/rejected": -350.29937744140625,
413
- "loss": 0.1211,
414
- "rewards/accuracies": 0.737500011920929,
415
- "rewards/chosen": -0.4048517346382141,
416
- "rewards/margins": 0.4419426918029785,
417
- "rewards/rejected": -0.8467944264411926,
418
  "step": 230
419
  },
420
  {
421
  "epoch": 0.5504587155963303,
422
- "grad_norm": 8.84744756928296,
423
  "learning_rate": 2.5e-07,
424
- "logits/chosen": -0.8606957197189331,
425
- "logits/rejected": -0.25019243359565735,
426
- "logps/chosen": -323.67840576171875,
427
- "logps/rejected": -341.2455749511719,
428
- "loss": 0.1198,
429
- "rewards/accuracies": 0.737500011920929,
430
- "rewards/chosen": -0.3614657521247864,
431
- "rewards/margins": 0.4374900758266449,
432
- "rewards/rejected": -0.7989557981491089,
433
  "step": 240
434
  },
435
  {
436
  "epoch": 0.573394495412844,
437
- "grad_norm": 8.762072273295908,
438
  "learning_rate": 2.2998577335247667e-07,
439
- "logits/chosen": -0.8679742813110352,
440
- "logits/rejected": -0.18818725645542145,
441
- "logps/chosen": -340.2317810058594,
442
- "logps/rejected": -349.81414794921875,
443
- "loss": 0.118,
444
- "rewards/accuracies": 0.78125,
445
- "rewards/chosen": -0.3643796145915985,
446
- "rewards/margins": 0.5293234586715698,
447
- "rewards/rejected": -0.8937031030654907,
448
  "step": 250
449
  },
450
  {
451
  "epoch": 0.573394495412844,
452
- "eval_logits/chosen": -0.8147245645523071,
453
- "eval_logits/rejected": -0.025059914216399193,
454
- "eval_logps/chosen": -322.4342041015625,
455
- "eval_logps/rejected": -343.3682556152344,
456
- "eval_loss": 0.11674168705940247,
457
- "eval_rewards/accuracies": 0.7543103694915771,
458
- "eval_rewards/chosen": -0.3734411895275116,
459
- "eval_rewards/margins": 0.5950165390968323,
460
- "eval_rewards/rejected": -0.9684576988220215,
461
- "eval_runtime": 95.6284,
462
- "eval_samples_per_second": 19.011,
463
- "eval_steps_per_second": 0.303,
464
  "step": 250
465
  },
466
  {
467
  "epoch": 0.5963302752293578,
468
- "grad_norm": 8.055173398595795,
469
  "learning_rate": 2.1010002624165524e-07,
470
- "logits/chosen": -0.5792064666748047,
471
- "logits/rejected": -0.028709029778838158,
472
- "logps/chosen": -319.5377197265625,
473
- "logps/rejected": -383.94610595703125,
474
- "loss": 0.1186,
475
- "rewards/accuracies": 0.78125,
476
- "rewards/chosen": -0.4882320463657379,
477
- "rewards/margins": 0.6072918176651001,
478
- "rewards/rejected": -1.0955239534378052,
479
  "step": 260
480
  },
481
  {
482
  "epoch": 0.6192660550458715,
483
- "grad_norm": 9.235774055264796,
484
  "learning_rate": 1.9047041344135043e-07,
485
- "logits/chosen": -0.37728264927864075,
486
- "logits/rejected": 0.017757948487997055,
487
- "logps/chosen": -318.92266845703125,
488
- "logps/rejected": -360.00250244140625,
489
- "loss": 0.1169,
490
- "rewards/accuracies": 0.8062499761581421,
491
- "rewards/chosen": -0.5286036729812622,
492
- "rewards/margins": 0.5529505014419556,
493
- "rewards/rejected": -1.0815541744232178,
494
  "step": 270
495
  },
496
  {
497
  "epoch": 0.6422018348623854,
498
- "grad_norm": 9.286061693337006,
499
  "learning_rate": 1.7122294549409482e-07,
500
- "logits/chosen": -0.7753764986991882,
501
- "logits/rejected": -0.038079630583524704,
502
- "logps/chosen": -314.3120422363281,
503
- "logps/rejected": -365.4776611328125,
504
- "loss": 0.1216,
505
- "rewards/accuracies": 0.78125,
506
- "rewards/chosen": -0.4131316542625427,
507
- "rewards/margins": 0.6036869883537292,
508
- "rewards/rejected": -1.016818642616272,
509
  "step": 280
510
  },
511
  {
512
  "epoch": 0.6651376146788991,
513
- "grad_norm": 10.391059671359317,
514
  "learning_rate": 1.524811797977383e-07,
515
- "logits/chosen": -0.6743412017822266,
516
- "logits/rejected": -0.05127597600221634,
517
- "logps/chosen": -326.2575378417969,
518
- "logps/rejected": -345.9309387207031,
519
- "loss": 0.1144,
520
- "rewards/accuracies": 0.737500011920929,
521
- "rewards/chosen": -0.3852667212486267,
522
- "rewards/margins": 0.5348750352859497,
523
- "rewards/rejected": -0.9201416969299316,
524
  "step": 290
525
  },
526
  {
527
  "epoch": 0.6880733944954128,
528
- "grad_norm": 11.142166437672913,
529
  "learning_rate": 1.3436542743979125e-07,
530
- "logits/chosen": -0.6289922595024109,
531
- "logits/rejected": -0.2742362916469574,
532
- "logps/chosen": -350.96099853515625,
533
- "logps/rejected": -345.9538879394531,
534
- "loss": 0.1236,
535
- "rewards/accuracies": 0.731249988079071,
536
- "rewards/chosen": -0.45011693239212036,
537
- "rewards/margins": 0.42082324624061584,
538
- "rewards/rejected": -0.8709400296211243,
539
  "step": 300
540
  },
541
  {
542
  "epoch": 0.6880733944954128,
543
- "eval_logits/chosen": -0.5284780263900757,
544
- "eval_logits/rejected": 0.2890760600566864,
545
- "eval_logps/chosen": -332.8935546875,
546
- "eval_logps/rejected": -351.01617431640625,
547
- "eval_loss": 0.11446455121040344,
548
- "eval_rewards/accuracies": 0.7629310488700867,
549
- "eval_rewards/chosen": -0.4780345857143402,
550
- "eval_rewards/margins": 0.5669029355049133,
551
- "eval_rewards/rejected": -1.0449376106262207,
552
- "eval_runtime": 94.2008,
553
- "eval_samples_per_second": 19.299,
554
- "eval_steps_per_second": 0.308,
555
  "step": 300
556
  },
557
  {
558
  "epoch": 0.7110091743119266,
559
- "grad_norm": 9.249175988688211,
560
  "learning_rate": 1.1699198087116588e-07,
561
- "logits/chosen": -0.5085445642471313,
562
- "logits/rejected": 0.12936155498027802,
563
- "logps/chosen": -332.48468017578125,
564
- "logps/rejected": -366.32489013671875,
565
- "loss": 0.1229,
566
- "rewards/accuracies": 0.75,
567
- "rewards/chosen": -0.5451949238777161,
568
- "rewards/margins": 0.4535568654537201,
569
- "rewards/rejected": -0.9987518191337585,
570
  "step": 310
571
  },
572
  {
573
  "epoch": 0.7339449541284404,
574
- "grad_norm": 9.319929398035963,
575
  "learning_rate": 1.00472367377196e-07,
576
- "logits/chosen": -0.37469834089279175,
577
- "logits/rejected": 0.34990087151527405,
578
- "logps/chosen": -323.309326171875,
579
- "logps/rejected": -343.5829772949219,
580
- "loss": 0.1184,
581
- "rewards/accuracies": 0.78125,
582
- "rewards/chosen": -0.49077337980270386,
583
- "rewards/margins": 0.6001543402671814,
584
- "rewards/rejected": -1.0909278392791748,
585
  "step": 320
586
  },
587
  {
588
  "epoch": 0.7568807339449541,
589
- "grad_norm": 10.87259656283017,
590
  "learning_rate": 8.49126331382102e-08,
591
- "logits/chosen": -0.20644304156303406,
592
- "logits/rejected": 0.2470076084136963,
593
- "logps/chosen": -329.26171875,
594
- "logps/rejected": -346.26666259765625,
595
- "loss": 0.1182,
596
- "rewards/accuracies": 0.7250000238418579,
597
- "rewards/chosen": -0.5473949909210205,
598
- "rewards/margins": 0.44427090883255005,
599
- "rewards/rejected": -0.9916658401489258,
600
  "step": 330
601
  },
602
  {
603
  "epoch": 0.7798165137614679,
604
- "grad_norm": 11.502524720737265,
605
  "learning_rate": 7.041266247556812e-08,
606
- "logits/chosen": -0.511077344417572,
607
- "logits/rejected": -0.07181791216135025,
608
- "logps/chosen": -338.0530700683594,
609
- "logps/rejected": -354.2392578125,
610
- "loss": 0.1129,
611
- "rewards/accuracies": 0.78125,
612
- "rewards/chosen": -0.5006048679351807,
613
- "rewards/margins": 0.4419478476047516,
614
- "rewards/rejected": -0.9425527453422546,
615
  "step": 340
616
  },
617
  {
618
  "epoch": 0.8027522935779816,
619
- "grad_norm": 10.417951906388708,
620
  "learning_rate": 5.706553665319955e-08,
621
- "logits/chosen": -0.4384083151817322,
622
- "logits/rejected": 0.22936472296714783,
623
- "logps/chosen": -327.1330261230469,
624
- "logps/rejected": -339.122314453125,
625
- "loss": 0.122,
626
- "rewards/accuracies": 0.762499988079071,
627
- "rewards/chosen": -0.526419460773468,
628
- "rewards/margins": 0.48818930983543396,
629
- "rewards/rejected": -1.0146087408065796,
630
  "step": 350
631
  },
632
  {
633
  "epoch": 0.8027522935779816,
634
- "eval_logits/chosen": -0.48056575655937195,
635
- "eval_logits/rejected": 0.33971619606018066,
636
- "eval_logps/chosen": -330.8570251464844,
637
- "eval_logps/rejected": -350.28302001953125,
638
- "eval_loss": 0.11267489194869995,
639
- "eval_rewards/accuracies": 0.767241358757019,
640
- "eval_rewards/chosen": -0.45766934752464294,
641
- "eval_rewards/margins": 0.5799364447593689,
642
- "eval_rewards/rejected": -1.0376057624816895,
643
- "eval_runtime": 94.5966,
644
- "eval_samples_per_second": 19.218,
645
- "eval_steps_per_second": 0.307,
646
  "step": 350
647
  },
648
  {
649
  "epoch": 0.8256880733944955,
650
- "grad_norm": 8.120253834632708,
651
  "learning_rate": 4.4956936350761005e-08,
652
- "logits/chosen": -0.412276029586792,
653
- "logits/rejected": 0.03293975815176964,
654
- "logps/chosen": -295.88397216796875,
655
- "logps/rejected": -349.67828369140625,
656
- "loss": 0.1126,
657
- "rewards/accuracies": 0.706250011920929,
658
- "rewards/chosen": -0.43563956022262573,
659
- "rewards/margins": 0.47577181458473206,
660
- "rewards/rejected": -0.9114114046096802,
661
  "step": 360
662
  },
663
  {
664
  "epoch": 0.8486238532110092,
665
- "grad_norm": 8.937083414679394,
666
  "learning_rate": 3.416459164418123e-08,
667
- "logits/chosen": -0.7006546258926392,
668
- "logits/rejected": -0.1087837815284729,
669
- "logps/chosen": -344.0120544433594,
670
- "logps/rejected": -357.1898498535156,
671
- "loss": 0.1124,
672
- "rewards/accuracies": 0.737500011920929,
673
- "rewards/chosen": -0.4442760944366455,
674
- "rewards/margins": 0.5018762350082397,
675
- "rewards/rejected": -0.9461522102355957,
676
  "step": 370
677
  },
678
  {
679
  "epoch": 0.8715596330275229,
680
- "grad_norm": 9.250415689561605,
681
  "learning_rate": 2.475778302439524e-08,
682
- "logits/chosen": -0.6538180112838745,
683
- "logits/rejected": 0.11514680087566376,
684
- "logps/chosen": -334.7962951660156,
685
- "logps/rejected": -353.5735778808594,
686
- "loss": 0.1165,
687
  "rewards/accuracies": 0.7749999761581421,
688
- "rewards/chosen": -0.3841826617717743,
689
- "rewards/margins": 0.5749909281730652,
690
- "rewards/rejected": -0.9591735601425171,
691
  "step": 380
692
  },
693
  {
694
  "epoch": 0.8944954128440367,
695
- "grad_norm": 9.532136069213038,
696
  "learning_rate": 1.6796896657433805e-08,
697
- "logits/chosen": -0.4097678065299988,
698
- "logits/rejected": 0.13332059979438782,
699
- "logps/chosen": -294.68280029296875,
700
- "logps/rejected": -319.02825927734375,
701
- "loss": 0.122,
702
- "rewards/accuracies": 0.7437499761581421,
703
- "rewards/chosen": -0.4272535443305969,
704
- "rewards/margins": 0.4699665606021881,
705
- "rewards/rejected": -0.8972201347351074,
706
  "step": 390
707
  },
708
  {
709
  "epoch": 0.9174311926605505,
710
- "grad_norm": 9.826819983657527,
711
  "learning_rate": 1.0333036740834855e-08,
712
- "logits/chosen": -0.3362836241722107,
713
- "logits/rejected": 0.09532846510410309,
714
- "logps/chosen": -262.51324462890625,
715
- "logps/rejected": -315.52349853515625,
716
- "loss": 0.12,
717
  "rewards/accuracies": 0.737500011920929,
718
- "rewards/chosen": -0.37953639030456543,
719
- "rewards/margins": 0.48354291915893555,
720
- "rewards/rejected": -0.8630792498588562,
721
  "step": 400
722
  },
723
  {
724
  "epoch": 0.9174311926605505,
725
- "eval_logits/chosen": -0.6159119606018066,
726
- "eval_logits/rejected": 0.19038529694080353,
727
- "eval_logps/chosen": -325.2745361328125,
728
- "eval_logps/rejected": -342.9399108886719,
729
- "eval_loss": 0.11256735771894455,
730
- "eval_rewards/accuracies": 0.75,
731
- "eval_rewards/chosen": -0.40184491872787476,
732
- "eval_rewards/margins": 0.5623298287391663,
733
- "eval_rewards/rejected": -0.9641746282577515,
734
- "eval_runtime": 93.497,
735
- "eval_samples_per_second": 19.444,
736
- "eval_steps_per_second": 0.31,
737
  "step": 400
738
  },
739
  {
740
  "epoch": 0.9403669724770642,
741
- "grad_norm": 9.708164546519836,
742
  "learning_rate": 5.4076974448211685e-09,
743
- "logits/chosen": -0.34120887517929077,
744
- "logits/rejected": 0.403463214635849,
745
- "logps/chosen": -314.7442321777344,
746
- "logps/rejected": -337.3968811035156,
747
- "loss": 0.1168,
748
- "rewards/accuracies": 0.7437499761581421,
749
- "rewards/chosen": -0.43792352080345154,
750
- "rewards/margins": 0.5555781126022339,
751
- "rewards/rejected": -0.9935015439987183,
752
  "step": 410
753
  },
754
  {
755
  "epoch": 0.963302752293578,
756
- "grad_norm": 10.889878191306106,
757
  "learning_rate": 2.052496544188487e-09,
758
- "logits/chosen": -0.31919345259666443,
759
- "logits/rejected": 0.3534119725227356,
760
- "logps/chosen": -306.3693542480469,
761
- "logps/rejected": -349.92462158203125,
762
- "loss": 0.1147,
763
- "rewards/accuracies": 0.768750011920929,
764
- "rewards/chosen": -0.4693039059638977,
765
- "rewards/margins": 0.5711686015129089,
766
- "rewards/rejected": -1.040472388267517,
767
  "step": 420
768
  },
769
  {
770
  "epoch": 0.9862385321100917,
771
- "grad_norm": 9.153719920278654,
772
  "learning_rate": 2.889724508297886e-10,
773
- "logits/chosen": -0.556784451007843,
774
- "logits/rejected": 0.2818067669868469,
775
- "logps/chosen": -345.05450439453125,
776
- "logps/rejected": -335.64605712890625,
777
- "loss": 0.1155,
778
- "rewards/accuracies": 0.7749999761581421,
779
- "rewards/chosen": -0.41879063844680786,
780
- "rewards/margins": 0.49730420112609863,
781
- "rewards/rejected": -0.9160947799682617,
782
  "step": 430
783
  },
784
  {
785
  "epoch": 1.0,
786
  "step": 436,
787
  "total_flos": 0.0,
788
- "train_loss": 0.13197535372108493,
789
- "train_runtime": 11523.0815,
790
- "train_samples_per_second": 4.839,
791
  "train_steps_per_second": 0.038
792
  }
793
  ],
 
10
  "log_history": [
11
  {
12
  "epoch": 0.022935779816513763,
13
+ "grad_norm": 1.6432102724209243,
14
  "learning_rate": 1.1363636363636363e-07,
15
+ "logits/chosen": -2.619717597961426,
16
+ "logits/rejected": -2.552694320678711,
17
+ "logps/chosen": -265.4180908203125,
18
+ "logps/rejected": -236.1053924560547,
19
+ "loss": 0.0154,
20
+ "rewards/accuracies": 0.3812499940395355,
21
+ "rewards/chosen": 0.00027717588818632066,
22
+ "rewards/margins": 3.1426665373146534e-05,
23
+ "rewards/rejected": 0.0002457492519170046,
24
  "step": 10
25
  },
26
  {
27
  "epoch": 0.045871559633027525,
28
+ "grad_norm": 1.5125635433247804,
29
  "learning_rate": 2.2727272727272726e-07,
30
+ "logits/chosen": -2.6578421592712402,
31
+ "logits/rejected": -2.5760762691497803,
32
+ "logps/chosen": -298.83837890625,
33
+ "logps/rejected": -274.30596923828125,
34
+ "loss": 0.0155,
35
+ "rewards/accuracies": 0.606249988079071,
36
+ "rewards/chosen": -0.0005307936808094382,
37
+ "rewards/margins": 0.0014294281136244535,
38
+ "rewards/rejected": -0.0019602221436798573,
39
  "step": 20
40
  },
41
  {
42
  "epoch": 0.06880733944954129,
43
+ "grad_norm": 1.3722478894206875,
44
  "learning_rate": 3.4090909090909085e-07,
45
+ "logits/chosen": -2.6761069297790527,
46
+ "logits/rejected": -2.602224826812744,
47
+ "logps/chosen": -290.43157958984375,
48
+ "logps/rejected": -234.3854522705078,
49
+ "loss": 0.0146,
50
+ "rewards/accuracies": 0.706250011920929,
51
+ "rewards/chosen": 0.00901214499026537,
52
+ "rewards/margins": 0.013047738000750542,
53
+ "rewards/rejected": -0.004035593010485172,
54
  "step": 30
55
  },
56
  {
57
  "epoch": 0.09174311926605505,
58
+ "grad_norm": 1.2692399849923888,
59
  "learning_rate": 4.545454545454545e-07,
60
+ "logits/chosen": -2.6601767539978027,
61
+ "logits/rejected": -2.610305070877075,
62
+ "logps/chosen": -280.9813537597656,
63
+ "logps/rejected": -267.60333251953125,
64
+ "loss": 0.013,
65
+ "rewards/accuracies": 0.6937500238418579,
66
+ "rewards/chosen": 0.04187846928834915,
67
+ "rewards/margins": 0.0399814248085022,
68
+ "rewards/rejected": 0.0018970475066453218,
69
  "step": 40
70
  },
71
  {
72
  "epoch": 0.11467889908256881,
73
+ "grad_norm": 1.4063277329047579,
74
  "learning_rate": 4.997110275491701e-07,
75
+ "logits/chosen": -2.6256864070892334,
76
+ "logits/rejected": -2.6204254627227783,
77
+ "logps/chosen": -289.7392578125,
78
+ "logps/rejected": -299.10064697265625,
79
+ "loss": 0.012,
80
+ "rewards/accuracies": 0.706250011920929,
81
+ "rewards/chosen": 0.06511950492858887,
82
+ "rewards/margins": 0.06428412348031998,
83
+ "rewards/rejected": 0.0008353829616680741,
84
  "step": 50
85
  },
86
  {
87
  "epoch": 0.11467889908256881,
88
+ "eval_logits/chosen": -2.578613519668579,
89
+ "eval_logits/rejected": -2.502546787261963,
90
+ "eval_logps/chosen": -277.33428955078125,
91
+ "eval_logps/rejected": -247.25254821777344,
92
+ "eval_loss": 0.01130068488419056,
93
+ "eval_rewards/accuracies": 0.6982758641242981,
94
+ "eval_rewards/chosen": 0.0775580108165741,
95
+ "eval_rewards/margins": 0.0848592072725296,
96
+ "eval_rewards/rejected": -0.007301207631826401,
97
+ "eval_runtime": 93.3854,
98
+ "eval_samples_per_second": 19.468,
99
+ "eval_steps_per_second": 0.311,
100
  "step": 50
101
  },
102
  {
103
  "epoch": 0.13761467889908258,
104
+ "grad_norm": 1.422544468484653,
105
  "learning_rate": 4.979475034558115e-07,
106
+ "logits/chosen": -2.580706834793091,
107
+ "logits/rejected": -2.525763750076294,
108
+ "logps/chosen": -284.323486328125,
109
+ "logps/rejected": -259.6044006347656,
110
+ "loss": 0.0112,
111
+ "rewards/accuracies": 0.6937500238418579,
112
+ "rewards/chosen": 0.05161212012171745,
113
+ "rewards/margins": 0.07209066301584244,
114
+ "rewards/rejected": -0.020478537306189537,
115
  "step": 60
116
  },
117
  {
118
  "epoch": 0.16055045871559634,
119
+ "grad_norm": 1.4554279086187263,
120
  "learning_rate": 4.945923025551788e-07,
121
+ "logits/chosen": -2.5086066722869873,
122
+ "logits/rejected": -2.471884250640869,
123
+ "logps/chosen": -319.5674743652344,
124
+ "logps/rejected": -265.2959899902344,
125
+ "loss": 0.0115,
126
+ "rewards/accuracies": 0.737500011920929,
127
+ "rewards/chosen": 0.08003517985343933,
128
+ "rewards/margins": 0.11891887336969376,
129
+ "rewards/rejected": -0.038883693516254425,
130
  "step": 70
131
  },
132
  {
133
  "epoch": 0.1834862385321101,
134
+ "grad_norm": 1.3828352586845378,
135
  "learning_rate": 4.896669632591651e-07,
136
+ "logits/chosen": -2.480229616165161,
137
+ "logits/rejected": -2.3888580799102783,
138
+ "logps/chosen": -279.42529296875,
139
+ "logps/rejected": -256.35528564453125,
140
+ "loss": 0.0109,
141
+ "rewards/accuracies": 0.7124999761581421,
142
+ "rewards/chosen": 0.06424586474895477,
143
+ "rewards/margins": 0.10123773664236069,
144
+ "rewards/rejected": -0.036991871893405914,
145
  "step": 80
146
  },
147
  {
148
  "epoch": 0.20642201834862386,
149
+ "grad_norm": 1.341734639019618,
150
  "learning_rate": 4.832031033425662e-07,
151
+ "logits/chosen": -2.430302381515503,
152
+ "logits/rejected": -2.3838858604431152,
153
+ "logps/chosen": -280.6355895996094,
154
+ "logps/rejected": -252.35107421875,
155
+ "loss": 0.0103,
156
+ "rewards/accuracies": 0.7124999761581421,
157
+ "rewards/chosen": 0.06957819312810898,
158
+ "rewards/margins": 0.12507636845111847,
159
+ "rewards/rejected": -0.0554981604218483,
160
  "step": 90
161
  },
162
  {
163
  "epoch": 0.22935779816513763,
164
+ "grad_norm": 1.503281183515388,
165
  "learning_rate": 4.752422169756047e-07,
166
+ "logits/chosen": -2.392822265625,
167
+ "logits/rejected": -2.337982654571533,
168
+ "logps/chosen": -269.06829833984375,
169
+ "logps/rejected": -280.2345886230469,
170
+ "loss": 0.0111,
171
+ "rewards/accuracies": 0.699999988079071,
172
+ "rewards/chosen": 0.033564209938049316,
173
+ "rewards/margins": 0.11662591993808746,
174
+ "rewards/rejected": -0.08306171000003815,
175
  "step": 100
176
  },
177
  {
178
  "epoch": 0.22935779816513763,
179
+ "eval_logits/chosen": -2.445643424987793,
180
+ "eval_logits/rejected": -2.345205545425415,
181
+ "eval_logps/chosen": -281.08892822265625,
182
+ "eval_logps/rejected": -254.68817138671875,
183
+ "eval_loss": 0.01000975538045168,
184
+ "eval_rewards/accuracies": 0.7112069129943848,
185
+ "eval_rewards/chosen": 0.04001150280237198,
186
+ "eval_rewards/margins": 0.1216687560081482,
187
+ "eval_rewards/rejected": -0.08165725320577621,
188
+ "eval_runtime": 92.3022,
189
+ "eval_samples_per_second": 19.696,
190
+ "eval_steps_per_second": 0.314,
191
  "step": 100
192
  },
193
  {
194
  "epoch": 0.25229357798165136,
195
+ "grad_norm": 2.297190949384611,
196
  "learning_rate": 4.658354083558188e-07,
197
+ "logits/chosen": -2.466749668121338,
198
+ "logits/rejected": -2.361680030822754,
199
+ "logps/chosen": -263.682373046875,
200
+ "logps/rejected": -241.5611572265625,
201
+ "loss": 0.0113,
202
+ "rewards/accuracies": 0.675000011920929,
203
+ "rewards/chosen": 0.015057370066642761,
204
+ "rewards/margins": 0.09333699941635132,
205
+ "rewards/rejected": -0.07827961444854736,
206
  "step": 110
207
  },
208
  {
209
  "epoch": 0.27522935779816515,
210
+ "grad_norm": 1.422856756971901,
211
  "learning_rate": 4.550430636492389e-07,
212
+ "logits/chosen": -2.3867597579956055,
213
+ "logits/rejected": -2.3550314903259277,
214
+ "logps/chosen": -275.4966125488281,
215
+ "logps/rejected": -260.20843505859375,
216
+ "loss": 0.0103,
217
+ "rewards/accuracies": 0.699999988079071,
218
+ "rewards/chosen": 0.0032271749805659056,
219
+ "rewards/margins": 0.09067237377166748,
220
+ "rewards/rejected": -0.08744519203901291,
221
  "step": 120
222
  },
223
  {
224
  "epoch": 0.2981651376146789,
225
+ "grad_norm": 2.508161764096464,
226
  "learning_rate": 4.429344633468004e-07,
227
+ "logits/chosen": -2.4056849479675293,
228
+ "logits/rejected": -2.361074686050415,
229
+ "logps/chosen": -254.7312469482422,
230
+ "logps/rejected": -252.63455200195312,
231
+ "loss": 0.0105,
232
  "rewards/accuracies": 0.699999988079071,
233
+ "rewards/chosen": 0.014488674700260162,
234
+ "rewards/margins": 0.11470804363489151,
235
+ "rewards/rejected": -0.10021935403347015,
236
  "step": 130
237
  },
238
  {
239
  "epoch": 0.3211009174311927,
240
+ "grad_norm": 3.762306745768528,
241
  "learning_rate": 4.2958733752443187e-07,
242
+ "logits/chosen": -2.412415027618408,
243
+ "logits/rejected": -2.3416905403137207,
244
+ "logps/chosen": -273.1282043457031,
245
+ "logps/rejected": -231.5812225341797,
246
+ "loss": 0.0122,
247
+ "rewards/accuracies": 0.7250000238418579,
248
+ "rewards/chosen": 0.03056950867176056,
249
+ "rewards/margins": 0.10341660678386688,
250
+ "rewards/rejected": -0.07284711301326752,
251
  "step": 140
252
  },
253
  {
254
  "epoch": 0.3440366972477064,
255
+ "grad_norm": 1.323990928834917,
256
  "learning_rate": 4.150873668617898e-07,
257
+ "logits/chosen": -2.454942226409912,
258
+ "logits/rejected": -2.3723654747009277,
259
+ "logps/chosen": -268.99053955078125,
260
+ "logps/rejected": -247.30032348632812,
261
+ "loss": 0.0104,
262
+ "rewards/accuracies": 0.668749988079071,
263
+ "rewards/chosen": -0.02446950413286686,
264
+ "rewards/margins": 0.11665806919336319,
265
+ "rewards/rejected": -0.1411275714635849,
266
  "step": 150
267
  },
268
  {
269
  "epoch": 0.3440366972477064,
270
+ "eval_logits/chosen": -2.4970576763153076,
271
+ "eval_logits/rejected": -2.400646924972534,
272
+ "eval_logps/chosen": -286.011474609375,
273
+ "eval_logps/rejected": -260.7337646484375,
274
+ "eval_loss": 0.009798681363463402,
275
+ "eval_rewards/accuracies": 0.7284482717514038,
276
+ "eval_rewards/chosen": -0.009214152581989765,
277
+ "eval_rewards/margins": 0.13289883732795715,
278
+ "eval_rewards/rejected": -0.14211300015449524,
279
+ "eval_runtime": 93.2661,
280
+ "eval_samples_per_second": 19.493,
281
+ "eval_steps_per_second": 0.311,
282
  "step": 150
283
  },
284
  {
285
  "epoch": 0.3669724770642202,
286
+ "grad_norm": 2.7039803270775495,
287
  "learning_rate": 3.9952763262280397e-07,
288
+ "logits/chosen": -2.460155963897705,
289
+ "logits/rejected": -2.39618182182312,
290
+ "logps/chosen": -293.5860595703125,
291
+ "logps/rejected": -295.9964294433594,
292
+ "loss": 0.01,
293
+ "rewards/accuracies": 0.7437499761581421,
294
+ "rewards/chosen": 0.02965056337416172,
295
+ "rewards/margins": 0.1299286037683487,
296
+ "rewards/rejected": -0.10027804225683212,
297
  "step": 160
298
  },
299
  {
300
  "epoch": 0.38990825688073394,
301
+ "grad_norm": 1.3210709780236793,
302
  "learning_rate": 3.8300801912883414e-07,
303
+ "logits/chosen": -2.503695249557495,
304
+ "logits/rejected": -2.4564812183380127,
305
+ "logps/chosen": -267.4649353027344,
306
+ "logps/rejected": -279.4906921386719,
307
+ "loss": 0.0091,
308
+ "rewards/accuracies": 0.7437499761581421,
309
+ "rewards/chosen": 0.041359588503837585,
310
+ "rewards/margins": 0.13557755947113037,
311
+ "rewards/rejected": -0.09421796351671219,
312
  "step": 170
313
  },
314
  {
315
  "epoch": 0.41284403669724773,
316
+ "grad_norm": 1.4526636355174283,
317
  "learning_rate": 3.6563457256020884e-07,
318
+ "logits/chosen": -2.5071005821228027,
319
+ "logits/rejected": -2.4085752964019775,
320
+ "logps/chosen": -306.22930908203125,
321
+ "logps/rejected": -253.84475708007812,
322
+ "loss": 0.0096,
323
+ "rewards/accuracies": 0.75,
324
+ "rewards/chosen": -0.028713583946228027,
325
+ "rewards/margins": 0.12478481233119965,
326
+ "rewards/rejected": -0.15349839627742767,
327
  "step": 180
328
  },
329
  {
330
  "epoch": 0.43577981651376146,
331
+ "grad_norm": 4.8242221106390115,
332
  "learning_rate": 3.475188202022617e-07,
333
+ "logits/chosen": -2.414685010910034,
334
+ "logits/rejected": -2.3840391635894775,
335
+ "logps/chosen": -256.085205078125,
336
+ "logps/rejected": -276.09039306640625,
337
+ "loss": 0.0106,
338
+ "rewards/accuracies": 0.7875000238418579,
339
+ "rewards/chosen": 0.020293032750487328,
340
+ "rewards/margins": 0.13775238394737244,
341
+ "rewards/rejected": -0.11745934188365936,
342
  "step": 190
343
  },
344
  {
345
  "epoch": 0.45871559633027525,
346
+ "grad_norm": 1.2637397948323732,
347
  "learning_rate": 3.287770545059052e-07,
348
+ "logits/chosen": -2.5293562412261963,
349
+ "logits/rejected": -2.431912899017334,
350
+ "logps/chosen": -277.83734130859375,
351
+ "logps/rejected": -256.05206298828125,
352
+ "loss": 0.0096,
353
+ "rewards/accuracies": 0.7437499761581421,
354
+ "rewards/chosen": 0.004443052224814892,
355
+ "rewards/margins": 0.12640419602394104,
356
+ "rewards/rejected": -0.12196113169193268,
357
  "step": 200
358
  },
359
  {
360
  "epoch": 0.45871559633027525,
361
+ "eval_logits/chosen": -2.5115060806274414,
362
+ "eval_logits/rejected": -2.4205849170684814,
363
+ "eval_logps/chosen": -282.7939147949219,
364
+ "eval_logps/rejected": -258.3851013183594,
365
+ "eval_loss": 0.00931267999112606,
366
+ "eval_rewards/accuracies": 0.7887930870056152,
367
+ "eval_rewards/chosen": 0.022961637005209923,
368
+ "eval_rewards/margins": 0.14158831536769867,
369
+ "eval_rewards/rejected": -0.1186266764998436,
370
+ "eval_runtime": 92.3817,
371
+ "eval_samples_per_second": 19.679,
372
+ "eval_steps_per_second": 0.314,
373
  "step": 200
374
  },
375
  {
376
  "epoch": 0.481651376146789,
377
+ "grad_norm": 1.093362739669567,
378
  "learning_rate": 3.0952958655864954e-07,
379
+ "logits/chosen": -2.4653537273406982,
380
+ "logits/rejected": -2.448239803314209,
381
+ "logps/chosen": -271.6222229003906,
382
+ "logps/rejected": -265.7300109863281,
383
+ "loss": 0.0094,
384
+ "rewards/accuracies": 0.7749999761581421,
385
+ "rewards/chosen": 0.01682201772928238,
386
+ "rewards/margins": 0.13070014119148254,
387
+ "rewards/rejected": -0.11387811601161957,
388
  "step": 210
389
  },
390
  {
391
  "epoch": 0.5045871559633027,
392
+ "grad_norm": 1.5040749987763142,
393
  "learning_rate": 2.898999737583448e-07,
394
+ "logits/chosen": -2.50272798538208,
395
+ "logits/rejected": -2.4043869972229004,
396
+ "logps/chosen": -322.4222717285156,
397
+ "logps/rejected": -300.2145690917969,
398
+ "loss": 0.0091,
399
+ "rewards/accuracies": 0.7749999761581421,
400
+ "rewards/chosen": 0.0025803411845117807,
401
+ "rewards/margins": 0.15502406656742096,
402
+ "rewards/rejected": -0.15244372189044952,
403
  "step": 220
404
  },
405
  {
406
  "epoch": 0.5275229357798165,
407
+ "grad_norm": 1.2805773201998596,
408
  "learning_rate": 2.7001422664752333e-07,
409
+ "logits/chosen": -2.396461009979248,
410
+ "logits/rejected": -2.344364881515503,
411
+ "logps/chosen": -268.88946533203125,
412
+ "logps/rejected": -278.57122802734375,
413
+ "loss": 0.0093,
414
+ "rewards/accuracies": 0.7124999761581421,
415
+ "rewards/chosen": -0.009459306485950947,
416
+ "rewards/margins": 0.12005350738763809,
417
+ "rewards/rejected": -0.12951281666755676,
418
  "step": 230
419
  },
420
  {
421
  "epoch": 0.5504587155963303,
422
+ "grad_norm": 1.346452258119606,
423
  "learning_rate": 2.5e-07,
424
+ "logits/chosen": -2.471860885620117,
425
+ "logits/rejected": -2.379460573196411,
426
+ "logps/chosen": -288.55877685546875,
427
+ "logps/rejected": -274.3955078125,
428
+ "loss": 0.0096,
429
+ "rewards/accuracies": 0.7562500238418579,
430
+ "rewards/chosen": -0.010269422084093094,
431
+ "rewards/margins": 0.12018589675426483,
432
+ "rewards/rejected": -0.13045531511306763,
433
  "step": 240
434
  },
435
  {
436
  "epoch": 0.573394495412844,
437
+ "grad_norm": 1.1636311800865655,
438
  "learning_rate": 2.2998577335247667e-07,
439
+ "logits/chosen": -2.5101351737976074,
440
+ "logits/rejected": -2.4231343269348145,
441
+ "logps/chosen": -303.5893249511719,
442
+ "logps/rejected": -274.129638671875,
443
+ "loss": 0.0093,
444
+ "rewards/accuracies": 0.768750011920929,
445
+ "rewards/chosen": 0.0020445596892386675,
446
+ "rewards/margins": 0.13890263438224792,
447
+ "rewards/rejected": -0.13685807585716248,
448
  "step": 250
449
  },
450
  {
451
  "epoch": 0.573394495412844,
452
+ "eval_logits/chosen": -2.459127187728882,
453
+ "eval_logits/rejected": -2.365269899368286,
454
+ "eval_logps/chosen": -286.2547912597656,
455
+ "eval_logps/rejected": -263.3385925292969,
456
+ "eval_loss": 0.008932252414524555,
457
+ "eval_rewards/accuracies": 0.7844827771186829,
458
+ "eval_rewards/chosen": -0.011646694503724575,
459
+ "eval_rewards/margins": 0.1565149575471878,
460
+ "eval_rewards/rejected": -0.16816167533397675,
461
+ "eval_runtime": 93.0437,
462
+ "eval_samples_per_second": 19.539,
463
+ "eval_steps_per_second": 0.312,
464
  "step": 250
465
  },
466
  {
467
  "epoch": 0.5963302752293578,
468
+ "grad_norm": 1.1997721634582499,
469
  "learning_rate": 2.1010002624165524e-07,
470
+ "logits/chosen": -2.4458436965942383,
471
+ "logits/rejected": -2.4075264930725098,
472
+ "logps/chosen": -273.95843505859375,
473
+ "logps/rejected": -292.99114990234375,
474
+ "loss": 0.0097,
475
+ "rewards/accuracies": 0.793749988079071,
476
+ "rewards/chosen": -0.032438840717077255,
477
+ "rewards/margins": 0.1535351574420929,
478
+ "rewards/rejected": -0.18597400188446045,
479
  "step": 260
480
  },
481
  {
482
  "epoch": 0.6192660550458715,
483
+ "grad_norm": 1.2414114105136553,
484
  "learning_rate": 1.9047041344135043e-07,
485
+ "logits/chosen": -2.437638759613037,
486
+ "logits/rejected": -2.409536600112915,
487
+ "logps/chosen": -270.0658264160156,
488
+ "logps/rejected": -270.24798583984375,
489
+ "loss": 0.0092,
490
+ "rewards/accuracies": 0.768750011920929,
491
+ "rewards/chosen": -0.04003521427512169,
492
+ "rewards/margins": 0.14397411048412323,
493
+ "rewards/rejected": -0.1840093433856964,
494
  "step": 270
495
  },
496
  {
497
  "epoch": 0.6422018348623854,
498
+ "grad_norm": 1.288255010907884,
499
  "learning_rate": 1.7122294549409482e-07,
500
+ "logits/chosen": -2.5124118328094482,
501
+ "logits/rejected": -2.4503026008605957,
502
+ "logps/chosen": -273.12841796875,
503
+ "logps/rejected": -280.5234375,
504
+ "loss": 0.0098,
505
+ "rewards/accuracies": 0.8062499761581421,
506
+ "rewards/chosen": -0.0012958184815943241,
507
+ "rewards/margins": 0.165980726480484,
508
+ "rewards/rejected": -0.167276531457901,
509
  "step": 280
510
  },
511
  {
512
  "epoch": 0.6651376146788991,
513
+ "grad_norm": 1.3451525886655082,
514
  "learning_rate": 1.524811797977383e-07,
515
+ "logits/chosen": -2.494507312774658,
516
+ "logits/rejected": -2.4191977977752686,
517
+ "logps/chosen": -287.3388366699219,
518
+ "logps/rejected": -267.7834167480469,
519
+ "loss": 0.009,
520
+ "rewards/accuracies": 0.78125,
521
+ "rewards/chosen": 0.003920386545360088,
522
+ "rewards/margins": 0.1425868272781372,
523
+ "rewards/rejected": -0.13866642117500305,
524
  "step": 290
525
  },
526
  {
527
  "epoch": 0.6880733944954128,
528
+ "grad_norm": 1.3558837079563066,
529
  "learning_rate": 1.3436542743979125e-07,
530
+ "logits/chosen": -2.498835563659668,
531
+ "logits/rejected": -2.4762415885925293,
532
+ "logps/chosen": -306.97705078125,
533
+ "logps/rejected": -270.83367919921875,
534
+ "loss": 0.0096,
535
+ "rewards/accuracies": 0.71875,
536
+ "rewards/chosen": -0.010277172550559044,
537
+ "rewards/margins": 0.10946089029312134,
538
+ "rewards/rejected": -0.11973806470632553,
539
  "step": 300
540
  },
541
  {
542
  "epoch": 0.6880733944954128,
543
+ "eval_logits/chosen": -2.4814062118530273,
544
+ "eval_logits/rejected": -2.38907790184021,
545
+ "eval_logps/chosen": -285.9173278808594,
546
+ "eval_logps/rejected": -262.4115295410156,
547
+ "eval_loss": 0.008779831230640411,
548
+ "eval_rewards/accuracies": 0.7844827771186829,
549
+ "eval_rewards/chosen": -0.008272184059023857,
550
+ "eval_rewards/margins": 0.15061868727207184,
551
+ "eval_rewards/rejected": -0.15889087319374084,
552
+ "eval_runtime": 92.1897,
553
+ "eval_samples_per_second": 19.72,
554
+ "eval_steps_per_second": 0.315,
555
  "step": 300
556
  },
557
  {
558
  "epoch": 0.7110091743119266,
559
+ "grad_norm": 1.3381129170655774,
560
  "learning_rate": 1.1699198087116588e-07,
561
+ "logits/chosen": -2.515507221221924,
562
+ "logits/rejected": -2.4294919967651367,
563
+ "logps/chosen": -280.1512756347656,
564
+ "logps/rejected": -280.0994873046875,
565
+ "loss": 0.0094,
566
+ "rewards/accuracies": 0.706250011920929,
567
+ "rewards/chosen": -0.02186095342040062,
568
+ "rewards/margins": 0.11463706195354462,
569
+ "rewards/rejected": -0.13649800419807434,
570
  "step": 310
571
  },
572
  {
573
  "epoch": 0.7339449541284404,
574
+ "grad_norm": 1.338991121862378,
575
  "learning_rate": 1.00472367377196e-07,
576
+ "logits/chosen": -2.451615571975708,
577
+ "logits/rejected": -2.404048442840576,
578
+ "logps/chosen": -275.2835388183594,
579
+ "logps/rejected": -251.0447998046875,
580
+ "loss": 0.0093,
581
+ "rewards/accuracies": 0.737500011920929,
582
+ "rewards/chosen": -0.010515814647078514,
583
+ "rewards/margins": 0.15502998232841492,
584
+ "rewards/rejected": -0.16554580628871918,
585
  "step": 320
586
  },
587
  {
588
  "epoch": 0.7568807339449541,
589
+ "grad_norm": 1.3896883263831044,
590
  "learning_rate": 8.49126331382102e-08,
591
+ "logits/chosen": -2.4412286281585693,
592
+ "logits/rejected": -2.392331600189209,
593
+ "logps/chosen": -275.5787353515625,
594
+ "logps/rejected": -260.4464111328125,
595
+ "loss": 0.0095,
596
+ "rewards/accuracies": 0.731249988079071,
597
+ "rewards/chosen": -0.010565539821982384,
598
+ "rewards/margins": 0.12289756536483765,
599
+ "rewards/rejected": -0.13346309959888458,
600
  "step": 330
601
  },
602
  {
603
  "epoch": 0.7798165137614679,
604
+ "grad_norm": 1.360283909503132,
605
  "learning_rate": 7.041266247556812e-08,
606
+ "logits/chosen": -2.525031566619873,
607
+ "logits/rejected": -2.481128215789795,
608
+ "logps/chosen": -289.6778564453125,
609
+ "logps/rejected": -272.6752014160156,
610
+ "loss": 0.009,
611
+ "rewards/accuracies": 0.75,
612
+ "rewards/chosen": -0.016852576285600662,
613
+ "rewards/margins": 0.11005936563014984,
614
+ "rewards/rejected": -0.1269119530916214,
615
  "step": 340
616
  },
617
  {
618
  "epoch": 0.8027522935779816,
619
+ "grad_norm": 1.3078891508366295,
620
  "learning_rate": 5.706553665319955e-08,
621
+ "logits/chosen": -2.501155138015747,
622
+ "logits/rejected": -2.399773597717285,
623
+ "logps/chosen": -277.02081298828125,
624
+ "logps/rejected": -253.2749786376953,
625
+ "loss": 0.0096,
626
+ "rewards/accuracies": 0.737500011920929,
627
+ "rewards/chosen": -0.025297433137893677,
628
+ "rewards/margins": 0.13083800673484802,
629
+ "rewards/rejected": -0.1561354398727417,
630
  "step": 350
631
  },
632
  {
633
  "epoch": 0.8027522935779816,
634
+ "eval_logits/chosen": -2.4845948219299316,
635
+ "eval_logits/rejected": -2.390641927719116,
636
+ "eval_logps/chosen": -285.5014343261719,
637
+ "eval_logps/rejected": -262.4816589355469,
638
+ "eval_loss": 0.008705741725862026,
639
+ "eval_rewards/accuracies": 0.7801724076271057,
640
+ "eval_rewards/chosen": -0.004113705363124609,
641
+ "eval_rewards/margins": 0.15547847747802734,
642
+ "eval_rewards/rejected": -0.15959219634532928,
643
+ "eval_runtime": 93.024,
644
+ "eval_samples_per_second": 19.543,
645
+ "eval_steps_per_second": 0.312,
646
  "step": 350
647
  },
648
  {
649
  "epoch": 0.8256880733944955,
650
+ "grad_norm": 1.1167320443463302,
651
  "learning_rate": 4.4956936350761005e-08,
652
+ "logits/chosen": -2.4814512729644775,
653
+ "logits/rejected": -2.4325201511383057,
654
+ "logps/chosen": -252.41159057617188,
655
+ "logps/rejected": -271.5174865722656,
656
+ "loss": 0.0088,
657
+ "rewards/accuracies": 0.737500011920929,
658
+ "rewards/chosen": -0.0009152599377557635,
659
+ "rewards/margins": 0.128888800740242,
660
+ "rewards/rejected": -0.12980404496192932,
661
  "step": 360
662
  },
663
  {
664
  "epoch": 0.8486238532110092,
665
+ "grad_norm": 1.2531440295822198,
666
  "learning_rate": 3.416459164418123e-08,
667
+ "logits/chosen": -2.5166125297546387,
668
+ "logits/rejected": -2.4559273719787598,
669
+ "logps/chosen": -299.89935302734375,
670
+ "logps/rejected": -276.8605041503906,
671
+ "loss": 0.0087,
672
+ "rewards/accuracies": 0.7437499761581421,
673
+ "rewards/chosen": -0.003148370888084173,
674
+ "rewards/margins": 0.13971056044101715,
675
+ "rewards/rejected": -0.14285892248153687,
676
  "step": 370
677
  },
678
  {
679
  "epoch": 0.8715596330275229,
680
+ "grad_norm": 1.494922388543688,
681
  "learning_rate": 2.475778302439524e-08,
682
+ "logits/chosen": -2.4861674308776855,
683
+ "logits/rejected": -2.4356298446655273,
684
+ "logps/chosen": -296.1441650390625,
685
+ "logps/rejected": -272.86956787109375,
686
+ "loss": 0.009,
687
  "rewards/accuracies": 0.7749999761581421,
688
+ "rewards/chosen": 0.0023386895190924406,
689
+ "rewards/margins": 0.15447205305099487,
690
+ "rewards/rejected": -0.15213337540626526,
691
  "step": 380
692
  },
693
  {
694
  "epoch": 0.8944954128440367,
695
+ "grad_norm": 1.505735003806321,
696
  "learning_rate": 1.6796896657433805e-08,
697
+ "logits/chosen": -2.4833850860595703,
698
+ "logits/rejected": -2.381284236907959,
699
+ "logps/chosen": -252.93844604492188,
700
+ "logps/rejected": -242.3744659423828,
701
+ "loss": 0.0095,
702
+ "rewards/accuracies": 0.71875,
703
+ "rewards/chosen": -0.009809909388422966,
704
+ "rewards/margins": 0.1208723783493042,
705
+ "rewards/rejected": -0.13068227469921112,
706
  "step": 390
707
  },
708
  {
709
  "epoch": 0.9174311926605505,
710
+ "grad_norm": 1.3636138422869082,
711
  "learning_rate": 1.0333036740834855e-08,
712
+ "logits/chosen": -2.41188383102417,
713
+ "logits/rejected": -2.378366231918335,
714
+ "logps/chosen": -223.8693389892578,
715
+ "logps/rejected": -241.79000854492188,
716
+ "loss": 0.0093,
717
  "rewards/accuracies": 0.737500011920929,
718
+ "rewards/chosen": 0.006902736611664295,
719
+ "rewards/margins": 0.13264694809913635,
720
+ "rewards/rejected": -0.12574420869350433,
721
  "step": 400
722
  },
723
  {
724
  "epoch": 0.9174311926605505,
725
+ "eval_logits/chosen": -2.4834423065185547,
726
+ "eval_logits/rejected": -2.389298439025879,
727
+ "eval_logps/chosen": -285.1916809082031,
728
+ "eval_logps/rejected": -261.8979797363281,
729
+ "eval_loss": 0.008685999549925327,
730
+ "eval_rewards/accuracies": 0.7758620977401733,
731
+ "eval_rewards/chosen": -0.0010154928313568234,
732
+ "eval_rewards/margins": 0.15273970365524292,
733
+ "eval_rewards/rejected": -0.15375518798828125,
734
+ "eval_runtime": 92.569,
735
+ "eval_samples_per_second": 19.639,
736
+ "eval_steps_per_second": 0.313,
737
  "step": 400
738
  },
739
  {
740
  "epoch": 0.9403669724770642,
741
+ "grad_norm": 1.3075058243660094,
742
  "learning_rate": 5.4076974448211685e-09,
743
+ "logits/chosen": -2.419281005859375,
744
+ "logits/rejected": -2.368946075439453,
745
+ "logps/chosen": -271.1397705078125,
746
+ "logps/rejected": -253.4750213623047,
747
+ "loss": 0.009,
748
+ "rewards/accuracies": 0.7749999761581421,
749
+ "rewards/chosen": -0.0018791112815961242,
750
+ "rewards/margins": 0.15240374207496643,
751
+ "rewards/rejected": -0.1542828381061554,
752
  "step": 410
753
  },
754
  {
755
  "epoch": 0.963302752293578,
756
+ "grad_norm": 2.8175803183046364,
757
  "learning_rate": 2.052496544188487e-09,
758
+ "logits/chosen": -2.462890148162842,
759
+ "logits/rejected": -2.377373218536377,
760
+ "logps/chosen": -260.7967834472656,
761
+ "logps/rejected": -261.8900451660156,
762
+ "loss": 0.0093,
763
+ "rewards/accuracies": 0.762499988079071,
764
+ "rewards/chosen": -0.013578072190284729,
765
+ "rewards/margins": 0.14654859900474548,
766
+ "rewards/rejected": -0.16012665629386902,
767
  "step": 420
768
  },
769
  {
770
  "epoch": 0.9862385321100917,
771
+ "grad_norm": 1.3745801458045255,
772
  "learning_rate": 2.889724508297886e-10,
773
+ "logits/chosen": -2.4770209789276123,
774
+ "logits/rejected": -2.3631789684295654,
775
+ "logps/chosen": -303.4706115722656,
776
+ "logps/rejected": -256.81878662109375,
777
+ "loss": 0.0094,
778
+ "rewards/accuracies": 0.762499988079071,
779
+ "rewards/chosen": -0.0029516436625272036,
780
+ "rewards/margins": 0.12487111985683441,
781
+ "rewards/rejected": -0.12782277166843414,
782
  "step": 430
783
  },
784
  {
785
  "epoch": 1.0,
786
  "step": 436,
787
  "total_flos": 0.0,
788
+ "train_loss": 0.010263856175705927,
789
+ "train_runtime": 11602.0724,
790
+ "train_samples_per_second": 4.806,
791
  "train_steps_per_second": 0.038
792
  }
793
  ],
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3d6fef0b25cff78175dd3905d73113b24ac1239ca17c5ec75144eef77b24b3ac
3
  size 7608
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:88d7448eeb3d751f828527ef8c7443426ba045f042d16c505dec59d9f53b2f8d
3
  size 7608