Hugging Face's logo

wxzhang
/
dpo-selective-buffer-safeipo

Text Generation
Transformers
Safetensors
mistral
trl
dpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
wxzhang commited on
Commit
581934d
•
1 Parent(s): 98f36d5

Model save

Browse files
Files changed (8) hide show
  1. README.md +17 -14
  2. all_results.json +18 -18
  3. eval_results.json +14 -14
  4. model-00001-of-00003.safetensors +1 -1
  5. model-00002-of-00003.safetensors +1 -1
  6. model-00003-of-00003.safetensors +1 -1
  7. train_results.json +4 -4
  8. trainer_state.json +0 -0
README.md CHANGED
@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
- - Loss: 17200.3730
19
- - Rewards/chosen: -0.9463
20
- - Rewards/rejected: -1.0095
21
- - Rewards/accuracies: 0.6067
22
- - Rewards/margins: 0.0632
23
- - Rewards/safe Rewards: -0.9414
24
- - Rewards/unsafe Rewards: -0.9445
25
- - Logps/rejected: -294.3703
26
- - Logps/chosen: -319.7016
27
- - Logits/rejected: 2.3506
28
- - Logits/chosen: 1.1396
29
 
30
  ## Model description
31
 
@@ -62,9 +62,12 @@ The following hyperparameters were used during training:
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
- | 48756.7969 | 0.27 | 500 | 17836.0742 | -0.6517 | -0.6890 | 0.5689 | 0.0374 | -0.6482 | -0.6494 | -230.2809 | -260.7748 | -1.4278 | -1.8042 |
66
- | 45676.9531 | 0.54 | 1000 | 17429.8281 | -0.7819 | -0.8708 | 0.6481 | 0.0890 | -0.7754 | -0.7778 | -266.6385 | -286.8121 | 1.9950 | 0.8553 |
67
- | 40856.9313 | 0.81 | 1500 | 17213.6094 | -0.9648 | -1.0237 | 0.6079 | 0.0589 | -0.9605 | -0.9638 | -297.2072 | -323.3928 | 2.5177 | 1.2878 |
 
 
 
68
 
69
 
70
  ### Framework versions
 
15
 
16
  This model was trained from scratch on the None dataset.
17
  It achieves the following results on the evaluation set:
18
+ - Loss: 4322.0576
19
+ - Rewards/chosen: -0.9426
20
+ - Rewards/rejected: -1.0072
21
+ - Rewards/accuracies: 0.6033
22
+ - Rewards/margins: 0.0646
23
+ - Rewards/safe Rewards: -0.9377
24
+ - Rewards/unsafe Rewards: -0.9382
25
+ - Logps/rejected: -193.1814
26
+ - Logps/chosen: -224.6856
27
+ - Logits/rejected: -1.7714
28
+ - Logits/chosen: -1.9525
29
 
30
  ## Model description
31
 
 
62
 
63
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
64
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
65
+ | 13096.7359 | 0.16 | 300 | 4529.6733 | -0.3957 | -0.4772 | 0.6584 | 0.0815 | -0.3930 | -0.3956 | -140.1830 | -170.0027 | -2.1815 | -2.3195 |
66
+ | 11584.7875 | 0.32 | 600 | 4406.7134 | -0.8083 | -0.8819 | 0.6338 | 0.0736 | -0.8028 | -0.8050 | -180.6571 | -211.2575 | -1.7938 | -1.9934 |
67
+ | 10862.3484 | 0.48 | 900 | 4377.5635 | -0.8828 | -0.9530 | 0.6196 | 0.0701 | -0.8775 | -0.8778 | -187.7609 | -218.7140 | -1.7468 | -1.9377 |
68
+ | 11671.4219 | 0.65 | 1200 | 4346.4053 | -0.9811 | -1.0509 | 0.6158 | 0.0699 | -0.9764 | -0.9768 | -197.5588 | -228.5369 | -1.6740 | -1.8665 |
69
+ | 10202.4125 | 0.81 | 1500 | 4320.9878 | -0.9655 | -1.0271 | 0.6023 | 0.0617 | -0.9611 | -0.9618 | -195.1794 | -226.9775 | -1.7645 | -1.9420 |
70
+ | 11785.8336 | 0.97 | 1800 | 4320.8208 | -0.9417 | -1.0065 | 0.6027 | 0.0648 | -0.9369 | -0.9373 | -193.1151 | -224.6014 | -1.7745 | -1.9550 |
71
 
72
 
73
  ### Framework versions
all_results.json CHANGED
@@ -1,23 +1,23 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": 1.1395671367645264,
4
- "eval_logits/rejected": 2.3505630493164062,
5
- "eval_logps/chosen": -319.7015686035156,
6
- "eval_logps/rejected": -294.37030029296875,
7
- "eval_loss": 17200.373046875,
8
- "eval_rewards/accuracies": 0.6067279577255249,
9
- "eval_rewards/chosen": -0.946313202381134,
10
- "eval_rewards/margins": 0.0631820559501648,
11
- "eval_rewards/rejected": -1.0094952583312988,
12
- "eval_rewards/safe_rewards": -0.9414083957672119,
13
- "eval_rewards/unsafe_rewards": -0.944499135017395,
14
- "eval_runtime": 1062.2335,
15
  "eval_samples": 33044,
16
- "eval_samples_per_second": 31.108,
17
- "eval_steps_per_second": 0.972,
18
- "train_loss": 47137.349905812705,
19
- "train_runtime": 22393.3393,
20
  "train_samples": 59478,
21
- "train_samples_per_second": 2.656,
22
- "train_steps_per_second": 0.083
23
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -1.9525097608566284,
4
+ "eval_logits/rejected": -1.7713532447814941,
5
+ "eval_logps/chosen": -224.6856231689453,
6
+ "eval_logps/rejected": -193.18142700195312,
7
+ "eval_loss": 4322.0576171875,
8
+ "eval_rewards/accuracies": 0.6033397912979126,
9
+ "eval_rewards/chosen": -0.9425504207611084,
10
+ "eval_rewards/margins": 0.06460745632648468,
11
+ "eval_rewards/rejected": -1.0071579217910767,
12
+ "eval_rewards/safe_rewards": -0.9376580119132996,
13
+ "eval_rewards/unsafe_rewards": -0.9381570816040039,
14
+ "eval_runtime": 993.6614,
15
  "eval_samples": 33044,
16
+ "eval_samples_per_second": 33.255,
17
+ "eval_steps_per_second": 1.04,
18
+ "train_loss": 11692.422362755651,
19
+ "train_runtime": 23508.9776,
20
  "train_samples": 59478,
21
+ "train_samples_per_second": 2.53,
22
+ "train_steps_per_second": 0.079
23
  }
eval_results.json CHANGED
@@ -1,18 +1,18 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": 1.1395671367645264,
4
- "eval_logits/rejected": 2.3505630493164062,
5
- "eval_logps/chosen": -319.7015686035156,
6
- "eval_logps/rejected": -294.37030029296875,
7
- "eval_loss": 17200.373046875,
8
- "eval_rewards/accuracies": 0.6067279577255249,
9
- "eval_rewards/chosen": -0.946313202381134,
10
- "eval_rewards/margins": 0.0631820559501648,
11
- "eval_rewards/rejected": -1.0094952583312988,
12
- "eval_rewards/safe_rewards": -0.9414083957672119,
13
- "eval_rewards/unsafe_rewards": -0.944499135017395,
14
- "eval_runtime": 1062.2335,
15
  "eval_samples": 33044,
16
- "eval_samples_per_second": 31.108,
17
- "eval_steps_per_second": 0.972
18
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "eval_logits/chosen": -1.9525097608566284,
4
+ "eval_logits/rejected": -1.7713532447814941,
5
+ "eval_logps/chosen": -224.6856231689453,
6
+ "eval_logps/rejected": -193.18142700195312,
7
+ "eval_loss": 4322.0576171875,
8
+ "eval_rewards/accuracies": 0.6033397912979126,
9
+ "eval_rewards/chosen": -0.9425504207611084,
10
+ "eval_rewards/margins": 0.06460745632648468,
11
+ "eval_rewards/rejected": -1.0071579217910767,
12
+ "eval_rewards/safe_rewards": -0.9376580119132996,
13
+ "eval_rewards/unsafe_rewards": -0.9381570816040039,
14
+ "eval_runtime": 993.6614,
15
  "eval_samples": 33044,
16
+ "eval_samples_per_second": 33.255,
17
+ "eval_steps_per_second": 1.04
18
  }
model-00001-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0cae3e9c2f07843bc94eb9798008f8926de896544008b976a659bc33c9ac1466
3
  size 4943162336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4150476808adc0b1f0332bef1db99919c14d4fdf7dbd79e3e92f2a46c1c2ba25
3
  size 4943162336
model-00002-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:314dd3ad4fa7bfe6ed8ee89722c9965a1117ac67b1818d4aa9e0fd47877a9a67
3
  size 4999819336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5bbee3c9f21058e85d167ecc2f7455822337aff3d60a54b51f38160452ba85e
3
  size 4999819336
model-00003-of-00003.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:194ddb5714d660dc426b40b826585d0cbb48b92a2cd27fc443f9b2751b77e61c
3
  size 4540516344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de513fed91b9adbf5bd450a2207739d475b951af82255b2144f7e2a464f8f004
3
  size 4540516344
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 47137.349905812705,
4
- "train_runtime": 22393.3393,
5
  "train_samples": 59478,
6
- "train_samples_per_second": 2.656,
7
- "train_steps_per_second": 0.083
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 11692.422362755651,
4
+ "train_runtime": 23508.9776,
5
  "train_samples": 59478,
6
+ "train_samples_per_second": 2.53,
7
+ "train_steps_per_second": 0.079
8
  }
trainer_state.json CHANGED
The diff for this file is too large to render. See raw diff