Model save

README.md

@@ -2,15 +2,9 @@
 license: mit
 base_model: HuggingFaceH4/mistral-7b-sft-beta
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: DPO-Zephyr-7B
   results: []
@@ -21,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # DPO-Zephyr-7B
 
-This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 ## Model description
 

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9874476987447699,
     "total_flos": 0.0,
-    "train_loss": 0.6490584389638092,
-    "train_runtime": 1634.5367,
+    "train_loss": 0.6490178754774191,
+    "train_runtime": 1630.8514,
     "train_samples": 15283,
-    "train_samples_per_second": 9.35,
+    "train_samples_per_second": 9.371,
     "train_steps_per_second": 0.036
 }

config.json

@@ -21,6 +21,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.2",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15e36704f67ca47c34862251973d343fa27833cd9298ee2ee3755ea705f6e5b3
+oid sha256:c102c7c24a4ffc9699e03c2bef20903e0d393c638be088dd4c10475225b2f91c
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cbe48df0ed8dcbfd88d5756391f17e4264d74e4e8a7ef2ef2206d71ccec48a7b
+oid sha256:5eec78610bcb027f2d9a6a0dbf2b4b18aa303873090b7bbac8f6a4193cd50c5b
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b254485f4a05f5a9c97e4bf1ee8e94a872ff6f62f44b7f451b681d44375cee17
+oid sha256:fc676b5fe1e6987a2d21ed941fe4d6ec069fa2a0ee9df004a9da8eb1462a4229
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9874476987447699,
     "total_flos": 0.0,
-    "train_loss": 0.6490584389638092,
-    "train_runtime": 1634.5367,
+    "train_loss": 0.6490178754774191,
+    "train_runtime": 1630.8514,
     "train_samples": 15283,
-    "train_samples_per_second": 9.35,
+    "train_samples_per_second": 9.371,
     "train_steps_per_second": 0.036
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.016736401673640166,
-      "grad_norm": 7.772396914178464,
+      "grad_norm": 7.772482099050406,
       "learning_rate": 8.333333333333333e-08,
       "logits/chosen": -2.807276487350464,
       "logits/rejected": -2.7759768962860107,
@@ -25,86 +25,86 @@
     },
     {
       "epoch": 0.16736401673640167,
-      "grad_norm": 7.458310251605103,
+      "grad_norm": 7.4630481490893485,
       "learning_rate": 4.930057285201027e-07,
-      "logits/chosen": -2.755532741546631,
-      "logits/rejected": -2.7461793422698975,
-      "logps/chosen": -271.9573974609375,
-      "logps/rejected": -260.7023620605469,
-      "loss": 0.6915,
-      "rewards/accuracies": 0.5243055820465088,
-      "rewards/chosen": 0.0051252637058496475,
-      "rewards/margins": 0.003539684694260359,
-      "rewards/rejected": 0.001585578778758645,
+      "logits/chosen": -2.7555835247039795,
+      "logits/rejected": -2.746177911758423,
+      "logps/chosen": -271.9296569824219,
+      "logps/rejected": -260.67138671875,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.5347222089767456,
+      "rewards/chosen": 0.0054025910794734955,
+      "rewards/margins": 0.0035073086619377136,
+      "rewards/rejected": 0.0018952824175357819,
       "step": 10
     },
     {
       "epoch": 0.33472803347280333,
-      "grad_norm": 6.720707588648352,
+      "grad_norm": 6.708300371266294,
       "learning_rate": 4.187457503795526e-07,
-      "logits/chosen": -2.786818027496338,
-      "logits/rejected": -2.7679667472839355,
-      "logps/chosen": -261.4557189941406,
-      "logps/rejected": -250.8779754638672,
+      "logits/chosen": -2.787097930908203,
+      "logits/rejected": -2.7682840824127197,
+      "logps/chosen": -261.45843505859375,
+      "logps/rejected": -250.8958740234375,
       "loss": 0.6745,
-      "rewards/accuracies": 0.703125,
-      "rewards/chosen": 0.03225986286997795,
-      "rewards/margins": 0.040614787489175797,
-      "rewards/rejected": -0.008354922756552696,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03223257511854172,
+      "rewards/margins": 0.040766604244709015,
+      "rewards/rejected": -0.008534022606909275,
       "step": 20
     },
     {
       "epoch": 0.502092050209205,
-      "grad_norm": 7.434878606336098,
+      "grad_norm": 7.409956341692702,
       "learning_rate": 2.8691164100062034e-07,
-      "logits/chosen": -2.7875802516937256,
-      "logits/rejected": -2.7685418128967285,
-      "logps/chosen": -293.7994079589844,
-      "logps/rejected": -254.1815948486328,
+      "logits/chosen": -2.7880468368530273,
+      "logits/rejected": -2.7689902782440186,
+      "logps/chosen": -293.8247985839844,
+      "logps/rejected": -254.217529296875,
       "loss": 0.6487,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": -0.02343321219086647,
-      "rewards/margins": 0.12680219113826752,
-      "rewards/rejected": -0.1502353996038437,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.023687291890382767,
+      "rewards/margins": 0.12690749764442444,
+      "rewards/rejected": -0.1505947858095169,
       "step": 30
     },
     {
       "epoch": 0.6694560669456067,
-      "grad_norm": 8.154579364951834,
+      "grad_norm": 8.167332409118467,
       "learning_rate": 1.4248369943086995e-07,
-      "logits/chosen": -2.760192394256592,
-      "logits/rejected": -2.737757444381714,
-      "logps/chosen": -264.1371154785156,
-      "logps/rejected": -255.595458984375,
+      "logits/chosen": -2.760864734649658,
+      "logits/rejected": -2.7383790016174316,
+      "logps/chosen": -264.1250915527344,
+      "logps/rejected": -255.5350799560547,
       "loss": 0.6354,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.11217646300792694,
-      "rewards/margins": 0.16067178547382355,
-      "rewards/rejected": -0.2728482186794281,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": -0.11205615103244781,
+      "rewards/margins": 0.16018818318843842,
+      "rewards/rejected": -0.27224433422088623,
       "step": 40
     },
     {
       "epoch": 0.8368200836820083,
-      "grad_norm": 8.586570040476488,
+      "grad_norm": 8.545155858110496,
       "learning_rate": 3.473909705816111e-08,
-      "logits/chosen": -2.77009654045105,
-      "logits/rejected": -2.7502076625823975,
-      "logps/chosen": -279.09228515625,
-      "logps/rejected": -287.27227783203125,
+      "logits/chosen": -2.770779848098755,
+      "logits/rejected": -2.750847339630127,
+      "logps/chosen": -279.1131286621094,
+      "logps/rejected": -287.27044677734375,
       "loss": 0.6202,
-      "rewards/accuracies": 0.621874988079071,
-      "rewards/chosen": -0.22083595395088196,
-      "rewards/margins": 0.1367621123790741,
-      "rewards/rejected": -0.35759806632995605,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.22104480862617493,
+      "rewards/margins": 0.13653476536273956,
+      "rewards/rejected": -0.3575795590877533,
       "step": 50
     },
     {
       "epoch": 0.9874476987447699,
       "step": 59,
       "total_flos": 0.0,
-      "train_loss": 0.6490584389638092,
-      "train_runtime": 1634.5367,
-      "train_samples_per_second": 9.35,
+      "train_loss": 0.6490178754774191,
+      "train_runtime": 1630.8514,
+      "train_samples_per_second": 9.371,
       "train_steps_per_second": 0.036
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d037ea8cb527c5399a0566fdabc99a692ba1350151e461ceec3204126d2dafc3
+oid sha256:cd4aa07115a1b0094354c437b369487eb676a555745cf463b02b3eaa8496dd8f
 size 6264