Model save

README.md

@@ -2,15 +2,9 @@
 license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 base_model: mistralai/Mistral-7B-v0.1
 model-index:
 - name: zephyr-7b-dpo-qlora-fsdp
@@ -22,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-qlora-fsdp
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-qlora](https://huggingface.co/alignment-handbook/zephyr-7b-sft-qlora) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6865
-- Rewards/chosen: 0.0331
-- Rewards/rejected: 0.0188
-- Rewards/accuracies: 0.5935
-- Rewards/margins: 0.0143
-- Logps/rejected: -257.1393
-- Logps/chosen: -276.4896
-- Logits/rejected: -2.3640
-- Logits/chosen: -2.4104
+- Loss: 0.6843
+- Rewards/chosen: 0.0234
+- Rewards/rejected: 0.0034
+- Rewards/accuracies: 0.6211
+- Rewards/margins: 0.0199
+- Logps/rejected: -260.8430
+- Logps/chosen: -258.9067
+- Logits/rejected: -2.4164
+- Logits/chosen: -2.4494
 
 ## Model description
 
@@ -52,16 +46,18 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-06
-- train_batch_size: 5
+- train_batch_size: 15
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
+- num_devices: 8
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 20
+- total_train_batch_size: 480
+- total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 0.01
+- num_epochs: 0.1
 
 ### Training results
 

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
-    "down_proj",
-    "v_proj",
+    "q_proj",
     "o_proj",
+    "down_proj",
+    "k_proj",
     "up_proj",
-    "q_proj",
-    "k_proj"
+    "gate_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f7925055af868d3b38756325d43d85883ad8c14753b88f93c708d2886161f80
+oid sha256:7a2d5fa611aeb51254da3b6144f743c1192dce3017f741b26f4cdf367d0100bb
 size 671150064

all_results.json

@@ -1,21 +1,21 @@
 {
-    "epoch": 0.01,
-    "eval_logits/chosen": -2.410409688949585,
-    "eval_logits/rejected": -2.363970994949341,
-    "eval_logps/chosen": -276.4896240234375,
-    "eval_logps/rejected": -257.1393127441406,
-    "eval_loss": 0.6865259408950806,
-    "eval_rewards/accuracies": 0.593500018119812,
-    "eval_rewards/chosen": 0.03312591835856438,
-    "eval_rewards/margins": 0.014345898292958736,
-    "eval_rewards/rejected": 0.018780020996928215,
-    "eval_runtime": 615.2299,
+    "epoch": 0.1,
+    "eval_logits/chosen": -2.449439287185669,
+    "eval_logits/rejected": -2.416422128677368,
+    "eval_logps/chosen": -258.90673828125,
+    "eval_logps/rejected": -260.843017578125,
+    "eval_loss": 0.6843283772468567,
+    "eval_rewards/accuracies": 0.62109375,
+    "eval_rewards/chosen": 0.023392992094159126,
+    "eval_rewards/margins": 0.019943276420235634,
+    "eval_rewards/rejected": 0.003449714742600918,
+    "eval_runtime": 144.9425,
     "eval_samples": 2000,
-    "eval_samples_per_second": 3.251,
-    "eval_steps_per_second": 0.406,
-    "train_loss": 0.6888245363389293,
-    "train_runtime": 439.9957,
+    "eval_samples_per_second": 13.799,
+    "eval_steps_per_second": 0.221,
+    "train_loss": 0.6879739944751446,
+    "train_runtime": 738.7034,
     "train_samples": 61135,
-    "train_samples_per_second": 1.389,
-    "train_steps_per_second": 0.07
+    "train_samples_per_second": 8.276,
+    "train_steps_per_second": 0.018
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 0.01,
-    "eval_logits/chosen": -2.410409688949585,
-    "eval_logits/rejected": -2.363970994949341,
-    "eval_logps/chosen": -276.4896240234375,
-    "eval_logps/rejected": -257.1393127441406,
-    "eval_loss": 0.6865259408950806,
-    "eval_rewards/accuracies": 0.593500018119812,
-    "eval_rewards/chosen": 0.03312591835856438,
-    "eval_rewards/margins": 0.014345898292958736,
-    "eval_rewards/rejected": 0.018780020996928215,
-    "eval_runtime": 615.2299,
+    "epoch": 0.1,
+    "eval_logits/chosen": -2.449439287185669,
+    "eval_logits/rejected": -2.416422128677368,
+    "eval_logps/chosen": -258.90673828125,
+    "eval_logps/rejected": -260.843017578125,
+    "eval_loss": 0.6843283772468567,
+    "eval_rewards/accuracies": 0.62109375,
+    "eval_rewards/chosen": 0.023392992094159126,
+    "eval_rewards/margins": 0.019943276420235634,
+    "eval_rewards/rejected": 0.003449714742600918,
+    "eval_runtime": 144.9425,
     "eval_samples": 2000,
-    "eval_samples_per_second": 3.251,
-    "eval_steps_per_second": 0.406
+    "eval_samples_per_second": 13.799,
+    "eval_steps_per_second": 0.221
 }

runs/Mar07_22-50-40_cccxc544/events.out.tfevents.1709869943.cccxc544.580669.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7082c4d428bfe48e310fa7b1025dac7ff000eca580c8abea515e181c224d9714
+size 12962

runs/Mar08_14-00-52_cccxc544/events.out.tfevents.1709924482.cccxc544.734409.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80bd2cf062bd929e2029872bce1ea0f91112afeda4c59bc369fb0cc6c827ffbd
+size 5478

runs/Mar08_14-12-12_cccxc542/events.out.tfevents.1709925175.cccxc542.465628.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08a6dc5383bb59b370a9b301fd1e8ea0188524008ea8a3bbfc19f8954fa27c75
+size 4801

runs/Mar08_14-17-17_cccxc544/events.out.tfevents.1709925468.cccxc544.737210.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51914d18d6a3ad3bebd45ef404fb2c51207b1cf788205fad758c13208e5900a1
+size 6501

runs/Mar08_14-17-17_cccxc544/events.out.tfevents.1709926351.cccxc544.737210.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32b05b4eeb189df79c4cf1cd4709802bd6f13350efde1948d3175e44e2118ef1
+size 815

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 0.01,
-    "train_loss": 0.6888245363389293,
-    "train_runtime": 439.9957,
+    "epoch": 0.1,
+    "train_loss": 0.6879739944751446,
+    "train_runtime": 738.7034,
     "train_samples": 61135,
-    "train_samples_per_second": 1.389,
-    "train_steps_per_second": 0.07
+    "train_samples_per_second": 8.276,
+    "train_steps_per_second": 0.018
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.010141490144761593,
+  "epoch": 0.10196078431372549,
   "eval_steps": 100,
-  "global_step": 31,
+  "global_step": 13,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "grad_norm": 2.421875,
-      "learning_rate": 1.25e-06,
-      "logits/chosen": -2.3689165115356445,
-      "logits/rejected": -2.3419089317321777,
-      "logps/chosen": -304.96429443359375,
-      "logps/rejected": -224.31954956054688,
+      "epoch": 0.01,
+      "grad_norm": 0.5078125,
+      "learning_rate": 2.5e-06,
+      "logits/chosen": -2.3996620178222656,
+      "logits/rejected": -2.364995002746582,
+      "logps/chosen": -288.2684326171875,
+      "logps/rejected": -260.847412109375,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,67 +24,37 @@
       "step": 1
     },
     {
-      "epoch": 0.0,
-      "grad_norm": 1.9296875,
-      "learning_rate": 4.415111107797445e-06,
-      "logits/chosen": -2.3774471282958984,
-      "logits/rejected": -2.358837127685547,
-      "logps/chosen": -267.6408386230469,
-      "logps/rejected": -221.9726104736328,
-      "loss": 0.6921,
-      "rewards/accuracies": 0.5166666507720947,
-      "rewards/chosen": 0.008927525021135807,
-      "rewards/margins": 0.002250629710033536,
-      "rewards/rejected": 0.006676895078271627,
+      "epoch": 0.08,
+      "grad_norm": 0.48046875,
+      "learning_rate": 8.628481651367876e-07,
+      "logits/chosen": -2.3491926193237305,
+      "logits/rejected": -2.307647705078125,
+      "logps/chosen": -264.3670349121094,
+      "logps/rejected": -252.78759765625,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.5314815640449524,
+      "rewards/chosen": 0.012712381780147552,
+      "rewards/margins": 0.01025653351098299,
+      "rewards/rejected": 0.0024558501318097115,
       "step": 10
     },
     {
-      "epoch": 0.01,
-      "grad_norm": 2.203125,
-      "learning_rate": 1.7829919182222752e-06,
-      "logits/chosen": -2.4560706615448,
-      "logits/rejected": -2.402303695678711,
-      "logps/chosen": -265.12762451171875,
-      "logps/rejected": -272.61566162109375,
-      "loss": 0.6889,
-      "rewards/accuracies": 0.5900000333786011,
-      "rewards/chosen": 0.028245043009519577,
-      "rewards/margins": 0.009032377041876316,
-      "rewards/rejected": 0.019212666898965836,
-      "step": 20
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 2.375,
-      "learning_rate": 1.6904105645142443e-08,
-      "logits/chosen": -2.3814165592193604,
-      "logits/rejected": -2.3470723628997803,
-      "logps/chosen": -304.08697509765625,
-      "logps/rejected": -281.0203552246094,
-      "loss": 0.6844,
-      "rewards/accuracies": 0.6299999952316284,
-      "rewards/chosen": 0.03472686558961868,
-      "rewards/margins": 0.019056813791394234,
-      "rewards/rejected": 0.015670055523514748,
-      "step": 30
-    },
-    {
-      "epoch": 0.01,
-      "step": 31,
+      "epoch": 0.1,
+      "step": 13,
       "total_flos": 0.0,
-      "train_loss": 0.6888245363389293,
-      "train_runtime": 439.9957,
-      "train_samples_per_second": 1.389,
-      "train_steps_per_second": 0.07
+      "train_loss": 0.6879739944751446,
+      "train_runtime": 738.7034,
+      "train_samples_per_second": 8.276,
+      "train_steps_per_second": 0.018
     }
   ],
   "logging_steps": 10,
-  "max_steps": 31,
+  "max_steps": 13,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
-  "train_batch_size": 5,
+  "train_batch_size": 15,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f21d618e5d6b3867a78c9fe4f30b09ded8634419a2c3a54b84735333c2487436
+oid sha256:3b632066f85ea8d2404c2331fdbd91457f0cc5397598475923b889072fa42749
 size 5112