Model save

README.md

@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 178.9674
-- Rewards/chosen: -0.2606
-- Rewards/rejected: -0.3242
-- Rewards/accuracies: 0.6077
-- Rewards/margins: 0.0636
-- Rewards/safe Rewards: -0.2588
-- Rewards/unsafe Rewards: -0.2621
-- Logps/rejected: -98.9557
-- Logps/chosen: -135.6514
-- Logits/rejected: -1.8071
-- Logits/chosen: -2.0526
+- Loss: 978.6271
+- Rewards/chosen: -0.2485
+- Rewards/rejected: -0.2964
+- Rewards/accuracies: 0.6600
+- Rewards/margins: 0.0478
+- Rewards/safe Rewards: -0.2450
+- Rewards/unsafe Rewards: -0.2427
+- Logps/rejected: -131.7681
+- Logps/chosen: -165.2006
+- Logits/rejected: -2.0856
+- Logits/chosen: -2.2494
 
 ## Model description
 
@@ -45,13 +45,13 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 4
+- train_batch_size: 2
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 4
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 64
+- total_train_batch_size: 32
 - total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
@@ -62,7 +62,13 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 500.1427      | 0.54  | 500  | 180.2062        | -0.2663        | -0.3383          | 0.6279             | 0.0720          | -0.2645              | -0.2672                | -99.2363       | -135.7646    | -1.7734         | -2.0265       |
+| 1504.5268     | 0.13  | 500  | 1034.4193       | -0.1746        | -0.2243          | 0.6809             | 0.0497          | -0.1711              | -0.1688                | -124.5656      | -157.8098    | -1.9343         | -2.1039       |
+| 1379.832      | 0.27  | 1000 | 1014.7580       | -0.1546        | -0.2013          | 0.6632             | 0.0467          | -0.1522              | -0.1501                | -122.2598      | -155.8100    | -2.1099         | -2.2674       |
+| 1404.9199     | 0.4   | 1500 | 997.0104        | -0.2219        | -0.2678          | 0.6678             | 0.0459          | -0.2189              | -0.2165                | -128.9146      | -162.5368    | -2.0713         | -2.2162       |
+| 1361.9422     | 0.53  | 2000 | 991.2021        | -0.2381        | -0.2863          | 0.6686             | 0.0481          | -0.2356              | -0.2330                | -130.7618      | -164.1645    | -2.1980         | -2.3435       |
+| 1406.6168     | 0.66  | 2500 | 981.6749        | -0.2153        | -0.2602          | 0.6503             | 0.0450          | -0.2126              | -0.2104                | -128.1535      | -161.8747    | -2.0826         | -2.2439       |
+| 1365.8523     | 0.8   | 3000 | 980.2808        | -0.2165        | -0.2645          | 0.6566             | 0.0481          | -0.2132              | -0.2111                | -128.5860      | -161.9975    | -2.0925         | -2.2557       |
+| 1242.352      | 0.93  | 3500 | 978.7930        | -0.2464        | -0.2943          | 0.6613             | 0.0479          | -0.2429              | -0.2405                | -131.5628      | -164.9901    | -2.0840         | -2.2476       |
 
 
 ### Framework versions

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.0526187419891357,
-    "eval_logits/rejected": -1.8070789575576782,
-    "eval_logps/chosen": -135.65139770507812,
-    "eval_logps/rejected": -98.95570373535156,
-    "eval_loss": 178.9673614501953,
-    "eval_rewards/accuracies": 0.6076960563659668,
-    "eval_rewards/chosen": -0.2606245279312134,
-    "eval_rewards/margins": 0.06359855830669403,
-    "eval_rewards/rejected": -0.3242231011390686,
-    "eval_rewards/safe_rewards": -0.2588219940662384,
-    "eval_rewards/unsafe_rewards": -0.26212161779403687,
-    "eval_runtime": 1126.1576,
-    "eval_samples": 33044,
-    "eval_samples_per_second": 29.342,
-    "eval_steps_per_second": 0.917,
-    "train_loss": 518.6781459188308,
-    "train_runtime": 19316.2719,
-    "train_samples": 59478,
-    "train_samples_per_second": 3.079,
-    "train_steps_per_second": 0.048
+    "eval_logits/chosen": -2.249445676803589,
+    "eval_logits/rejected": -2.085613965988159,
+    "eval_logps/chosen": -165.2006378173828,
+    "eval_logps/rejected": -131.76806640625,
+    "eval_loss": 978.6270751953125,
+    "eval_rewards/accuracies": 0.6600136756896973,
+    "eval_rewards/chosen": -0.24851088225841522,
+    "eval_rewards/margins": 0.04783935099840164,
+    "eval_rewards/rejected": -0.2963502109050751,
+    "eval_rewards/safe_rewards": -0.24501556158065796,
+    "eval_rewards/unsafe_rewards": -0.2426704317331314,
+    "eval_runtime": 2349.9274,
+    "eval_samples": 35044,
+    "eval_samples_per_second": 14.913,
+    "eval_steps_per_second": 0.466,
+    "train_loss": 1500.3740138506566,
+    "train_runtime": 40588.9258,
+    "train_samples": 120613,
+    "train_samples_per_second": 2.972,
+    "train_steps_per_second": 0.093
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.0526187419891357,
-    "eval_logits/rejected": -1.8070789575576782,
-    "eval_logps/chosen": -135.65139770507812,
-    "eval_logps/rejected": -98.95570373535156,
-    "eval_loss": 178.9673614501953,
-    "eval_rewards/accuracies": 0.6076960563659668,
-    "eval_rewards/chosen": -0.2606245279312134,
-    "eval_rewards/margins": 0.06359855830669403,
-    "eval_rewards/rejected": -0.3242231011390686,
-    "eval_rewards/safe_rewards": -0.2588219940662384,
-    "eval_rewards/unsafe_rewards": -0.26212161779403687,
-    "eval_runtime": 1126.1576,
-    "eval_samples": 33044,
-    "eval_samples_per_second": 29.342,
-    "eval_steps_per_second": 0.917
+    "eval_logits/chosen": -2.249445676803589,
+    "eval_logits/rejected": -2.085613965988159,
+    "eval_logps/chosen": -165.2006378173828,
+    "eval_logps/rejected": -131.76806640625,
+    "eval_loss": 978.6270751953125,
+    "eval_rewards/accuracies": 0.6600136756896973,
+    "eval_rewards/chosen": -0.24851088225841522,
+    "eval_rewards/margins": 0.04783935099840164,
+    "eval_rewards/rejected": -0.2963502109050751,
+    "eval_rewards/safe_rewards": -0.24501556158065796,
+    "eval_rewards/unsafe_rewards": -0.2426704317331314,
+    "eval_runtime": 2349.9274,
+    "eval_samples": 35044,
+    "eval_samples_per_second": 14.913,
+    "eval_steps_per_second": 0.466
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b251db184e1cdb90f90f481831b4377ac1b54acc518abe02dd58dcf3176cd316
+oid sha256:66ac9d413cabbb2c76c373a804e7399722f74af000828063d871e6b6058bd8c2
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e75f0abe3d933f209053af6a69a583e65387ef44885d0be97dfe4df94b16c733
+oid sha256:e930cd85138928e973fe4ea05bc40656880d53ed562dc4340a7a5331c45c49af
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b5033e55182234232af674e0a4d84397eb93b8a2956ed572bdd22146428de83
+oid sha256:c1bbe49aa0fb4bf303388388ba17a7836933dd8e445a9e048d71ee389b9399a0
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 518.6781459188308,
-    "train_runtime": 19316.2719,
-    "train_samples": 59478,
-    "train_samples_per_second": 3.079,
-    "train_steps_per_second": 0.048
+    "train_loss": 1500.3740138506566,
+    "train_runtime": 40588.9258,
+    "train_samples": 120613,
+    "train_samples_per_second": 2.972,
+    "train_steps_per_second": 0.093
 }