Model save

README.md

@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 4322.0576
-- Rewards/chosen: -0.9426
-- Rewards/rejected: -1.0072
-- Rewards/accuracies: 0.6033
-- Rewards/margins: 0.0646
-- Rewards/safe Rewards: -0.9377
-- Rewards/unsafe Rewards: -0.9382
-- Logps/rejected: -193.1814
-- Logps/chosen: -224.6856
-- Logits/rejected: -1.7714
-- Logits/chosen: -1.9525
+- Loss: 4449.9023
+- Rewards/chosen: -0.8766
+- Rewards/rejected: -0.9587
+- Rewards/accuracies: 0.6161
+- Rewards/margins: 0.0822
+- Rewards/safe Rewards: -0.8653
+- Rewards/unsafe Rewards: -0.8608
+- Logps/rejected: -198.0037
+- Logps/chosen: -228.0047
+- Logits/rejected: 1.7482
+- Logits/chosen: 0.9054
 
 ## Model description
 
@@ -45,13 +45,13 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 2
+- train_batch_size: 4
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 4
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 32
+- total_train_batch_size: 64
 - total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
@@ -62,12 +62,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 13096.7359    | 0.16  | 300  | 4529.6733       | -0.3957        | -0.4772          | 0.6584             | 0.0815          | -0.3930              | -0.3956                | -140.1830      | -170.0027    | -2.1815         | -2.3195       |
-| 11584.7875    | 0.32  | 600  | 4406.7134       | -0.8083        | -0.8819          | 0.6338             | 0.0736          | -0.8028              | -0.8050                | -180.6571      | -211.2575    | -1.7938         | -1.9934       |
-| 10862.3484    | 0.48  | 900  | 4377.5635       | -0.8828        | -0.9530          | 0.6196             | 0.0701          | -0.8775              | -0.8778                | -187.7609      | -218.7140    | -1.7468         | -1.9377       |
-| 11671.4219    | 0.65  | 1200 | 4346.4053       | -0.9811        | -1.0509          | 0.6158             | 0.0699          | -0.9764              | -0.9768                | -197.5588      | -228.5369    | -1.6740         | -1.8665       |
-| 10202.4125    | 0.81  | 1500 | 4320.9878       | -0.9655        | -1.0271          | 0.6023             | 0.0617          | -0.9611              | -0.9618                | -195.1794      | -226.9775    | -1.7645         | -1.9420       |
-| 11785.8336    | 0.97  | 1800 | 4320.8208       | -0.9417        | -1.0065          | 0.6027             | 0.0648          | -0.9369              | -0.9373                | -193.1151      | -224.6014    | -1.7745         | -1.9550       |
+| 5410.1973     | 0.27  | 500  | 4657.3340       | -0.6508        | -0.7493          | 0.6367             | 0.0984          | -0.6382              | -0.6354                | -177.0600      | -205.4323    | 0.6948          | -0.0099       |
+| 5634.6316     | 0.53  | 1000 | 4507.8945       | -0.8000        | -0.8748          | 0.6152             | 0.0748          | -0.7886              | -0.7846                | -189.6167      | -220.3491    | 1.1542          | 0.4120        |
+| 5749.5141     | 0.8   | 1500 | 4458.4429       | -0.8858        | -0.9723          | 0.6194             | 0.0865          | -0.8741              | -0.8700                | -199.3641      | -228.9305    | 1.9547          | 1.0718        |
 
 
 ### Framework versions

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -1.9525097608566284,
-    "eval_logits/rejected": -1.7713532447814941,
-    "eval_logps/chosen": -224.6856231689453,
-    "eval_logps/rejected": -193.18142700195312,
-    "eval_loss": 4322.0576171875,
-    "eval_rewards/accuracies": 0.6033397912979126,
-    "eval_rewards/chosen": -0.9425504207611084,
-    "eval_rewards/margins": 0.06460745632648468,
-    "eval_rewards/rejected": -1.0071579217910767,
-    "eval_rewards/safe_rewards": -0.9376580119132996,
-    "eval_rewards/unsafe_rewards": -0.9381570816040039,
-    "eval_runtime": 993.6614,
-    "eval_samples": 33044,
-    "eval_samples_per_second": 33.255,
-    "eval_steps_per_second": 1.04,
-    "train_loss": 11692.422362755651,
-    "train_runtime": 23508.9776,
-    "train_samples": 59478,
-    "train_samples_per_second": 2.53,
-    "train_steps_per_second": 0.079
+    "eval_logits/chosen": 0.9053679704666138,
+    "eval_logits/rejected": 1.7481720447540283,
+    "eval_logps/chosen": -228.00465393066406,
+    "eval_logps/rejected": -198.00367736816406,
+    "eval_loss": 4449.90234375,
+    "eval_rewards/accuracies": 0.616104006767273,
+    "eval_rewards/chosen": -0.8765509724617004,
+    "eval_rewards/margins": 0.08215557038784027,
+    "eval_rewards/rejected": -0.9587064981460571,
+    "eval_rewards/safe_rewards": -0.865267813205719,
+    "eval_rewards/unsafe_rewards": -0.860846996307373,
+    "eval_runtime": 2354.4025,
+    "eval_samples": 35044,
+    "eval_samples_per_second": 14.884,
+    "eval_steps_per_second": 0.466,
+    "train_loss": 5859.617769083399,
+    "train_runtime": 32772.3871,
+    "train_samples": 120613,
+    "train_samples_per_second": 3.68,
+    "train_steps_per_second": 0.057
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -1.9525097608566284,
-    "eval_logits/rejected": -1.7713532447814941,
-    "eval_logps/chosen": -224.6856231689453,
-    "eval_logps/rejected": -193.18142700195312,
-    "eval_loss": 4322.0576171875,
-    "eval_rewards/accuracies": 0.6033397912979126,
-    "eval_rewards/chosen": -0.9425504207611084,
-    "eval_rewards/margins": 0.06460745632648468,
-    "eval_rewards/rejected": -1.0071579217910767,
-    "eval_rewards/safe_rewards": -0.9376580119132996,
-    "eval_rewards/unsafe_rewards": -0.9381570816040039,
-    "eval_runtime": 993.6614,
-    "eval_samples": 33044,
-    "eval_samples_per_second": 33.255,
-    "eval_steps_per_second": 1.04
+    "eval_logits/chosen": 0.9053679704666138,
+    "eval_logits/rejected": 1.7481720447540283,
+    "eval_logps/chosen": -228.00465393066406,
+    "eval_logps/rejected": -198.00367736816406,
+    "eval_loss": 4449.90234375,
+    "eval_rewards/accuracies": 0.616104006767273,
+    "eval_rewards/chosen": -0.8765509724617004,
+    "eval_rewards/margins": 0.08215557038784027,
+    "eval_rewards/rejected": -0.9587064981460571,
+    "eval_rewards/safe_rewards": -0.865267813205719,
+    "eval_rewards/unsafe_rewards": -0.860846996307373,
+    "eval_runtime": 2354.4025,
+    "eval_samples": 35044,
+    "eval_samples_per_second": 14.884,
+    "eval_steps_per_second": 0.466
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4150476808adc0b1f0332bef1db99919c14d4fdf7dbd79e3e92f2a46c1c2ba25
+oid sha256:fd1ffd62dbd1638df8943c7aaecfbf0a36ab6c310ea7526c641642c038708374
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5bbee3c9f21058e85d167ecc2f7455822337aff3d60a54b51f38160452ba85e
+oid sha256:8857c80e3bd6f6cdf0577edd0c683a84339d37a7deb9cfb6435e798d3c8a7663
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de513fed91b9adbf5bd450a2207739d475b951af82255b2144f7e2a464f8f004
+oid sha256:ccdb216940fac7c77c3c865210942e9892bd76e4b1675024f8c1586b64f5e704
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 11692.422362755651,
-    "train_runtime": 23508.9776,
-    "train_samples": 59478,
-    "train_samples_per_second": 2.53,
-    "train_steps_per_second": 0.079
+    "train_loss": 5859.617769083399,
+    "train_runtime": 32772.3871,
+    "train_samples": 120613,
+    "train_samples_per_second": 3.68,
+    "train_steps_per_second": 0.057
 }