update model checkpoint

README.md

@@ -1,14 +1,11 @@
 ---
-license: gemma
 library_name: peft
 tags:
 - alignment-handbook
-- trl
-- sft
 - generated_from_trainer
-base_model: google/gemma-7b
 datasets:
 - llama-duo/synth_summarize_dataset_dedup
+base_model: google/gemma-7b
 model-index:
 - name: gemma7b-summarize-gpt4o-2k
   results: []
@@ -21,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on the llama-duo/synth_summarize_dataset_dedup dataset.
 It achieves the following results on the evaluation set:
-- Loss: 3.2983
+- Loss: 7.6472
 
 ## Model description
 
@@ -42,13 +39,13 @@ More information needed
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
 - train_batch_size: 4
-- eval_batch_size: 4
+- eval_batch_size: 2
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 2
+- num_devices: 8
 - gradient_accumulation_steps: 2
-- total_train_batch_size: 16
-- total_eval_batch_size: 8
+- total_train_batch_size: 64
+- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -56,24 +53,23 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss |
-|:-------------:|:-----:|:----:|:---------------:|
-| 23.4809       | 1.0   | 13   | 8.2506          |
-| 7.0034        | 2.0   | 26   | 3.5645          |
-| 1.9278        | 3.0   | 39   | 2.7831          |
-| 1.3284        | 4.0   | 52   | 2.7295          |
-| 1.046         | 5.0   | 65   | 2.7647          |
-| 0.8611        | 6.0   | 78   | 2.8629          |
-| 0.7242        | 7.0   | 91   | 3.0405          |
-| 0.6192        | 8.0   | 104  | 3.1549          |
-| 0.5312        | 9.0   | 117  | 3.3041          |
-| 0.5202        | 10.0  | 130  | 3.2983          |
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 47.9246       | 0.8571 | 3    | 15.4847         |
+| 38.5919       | 2.0    | 7    | 12.5499         |
+| 23.0632       | 2.8571 | 10   | 10.1538         |
+| 23.0632       | 4.0    | 14   | 8.4647          |
+| 19.8584       | 4.8571 | 17   | 8.0216          |
+| 19.1062       | 6.0    | 21   | 7.7569          |
+| 19.1062       | 6.8571 | 24   | 7.6779          |
+| 18.5688       | 8.0    | 28   | 7.6438          |
+| 18.5805       | 8.5714 | 30   | 7.6472          |
 
 
 ### Framework versions
 
 - PEFT 0.10.0
 - Transformers 4.40.0
-- Pytorch 2.2.1+cu121
+- Pytorch 2.1.2+cu121
 - Datasets 2.18.0
 - Tokenizers 0.19.1

adapter_config.json

@@ -20,13 +20,8 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
-    "up_proj",
-    "gate_proj",
     "v_proj",
-    "q_proj",
-    "k_proj",
-    "o_proj"
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d553ae6702fe97cb32e10a3ba799b7a9b102497c3e9a1c2320c4275af05e6c45
-size 50056096
+oid sha256:2f662f6233284b3834b115fddd86956ea1ff79d185cb8cc5f9d0a98aaae3fdcc
+size 6437384

all_results.json

@@ -1,14 +1,14 @@
 {
-    "epoch": 10.0,
-    "eval_loss": 3.298276901245117,
-    "eval_runtime": 2.0609,
+    "epoch": 8.571428571428571,
+    "eval_loss": 7.6471662521362305,
+    "eval_runtime": 0.2338,
     "eval_samples": 25,
-    "eval_samples_per_second": 4.852,
-    "eval_steps_per_second": 0.97,
-    "total_flos": 9.937675599478784e+16,
-    "train_loss": 4.9734316147290745,
-    "train_runtime": 870.6795,
-    "train_samples": 1874,
-    "train_samples_per_second": 2.366,
-    "train_steps_per_second": 0.149
+    "eval_samples_per_second": 42.78,
+    "eval_steps_per_second": 4.278,
+    "total_flos": 9.147533553041408e+16,
+    "train_loss": 23.272574106852215,
+    "train_runtime": 74.8732,
+    "train_samples": 2019,
+    "train_samples_per_second": 29.383,
+    "train_steps_per_second": 0.401
 }

config.json

@@ -23,9 +23,9 @@
     "_load_in_4bit": true,
     "_load_in_8bit": false,
     "bnb_4bit_compute_dtype": "bfloat16",
-    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_storage": "uint8",
     "bnb_4bit_quant_type": "nf4",
-    "bnb_4bit_use_double_quant": true,
+    "bnb_4bit_use_double_quant": false,
     "llm_int8_enable_fp32_cpu_offload": false,
     "llm_int8_has_fp16_weight": false,
     "llm_int8_skip_modules": null,

eval_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 10.0,
-    "eval_loss": 3.298276901245117,
-    "eval_runtime": 2.0609,
+    "epoch": 8.571428571428571,
+    "eval_loss": 7.6471662521362305,
+    "eval_runtime": 0.2338,
     "eval_samples": 25,
-    "eval_samples_per_second": 4.852,
-    "eval_steps_per_second": 0.97
+    "eval_samples_per_second": 42.78,
+    "eval_steps_per_second": 4.278
 }

runs/Jun13_05-45-11_gpu1-2/events.out.tfevents.1718228757.gpu1-2.1116744.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ee45360b016f76c2bf3bf0c1464de946a4da8cd773f7824814dbae4f55a1f7c
+size 9638

runs/Jun13_05-45-11_gpu1-2/events.out.tfevents.1718228832.gpu1-2.1116744.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1f8bda717b006d9c4b6efb9157ba3be0fb069bbbdb1125c0760a846c1ccab1f
+size 354

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 10.0,
-    "total_flos": 9.937675599478784e+16,
-    "train_loss": 4.9734316147290745,
-    "train_runtime": 870.6795,
-    "train_samples": 1874,
-    "train_samples_per_second": 2.366,
-    "train_steps_per_second": 0.149
+    "epoch": 8.571428571428571,
+    "total_flos": 9.147533553041408e+16,
+    "train_loss": 23.272574106852215,
+    "train_runtime": 74.8732,
+    "train_samples": 2019,
+    "train_samples_per_second": 29.383,
+    "train_steps_per_second": 0.401
 }

trainer_state.json

@@ -1,298 +1,150 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 10.0,
+  "epoch": 8.571428571428571,
   "eval_steps": 500,
-  "global_step": 130,
+  "global_step": 30,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.07692307692307693,
-      "grad_norm": 480.0,
-      "learning_rate": 1.5384615384615387e-05,
-      "loss": 49.667,
+      "epoch": 0.2857142857142857,
+      "grad_norm": 189.0,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 47.9246,
       "step": 1
     },
     {
-      "epoch": 0.38461538461538464,
-      "grad_norm": 260.0,
-      "learning_rate": 7.692307692307693e-05,
-      "loss": 40.4711,
-      "step": 5
-    },
-    {
-      "epoch": 0.7692307692307693,
-      "grad_norm": 26.75,
-      "learning_rate": 0.00015384615384615385,
-      "loss": 23.4809,
-      "step": 10
-    },
-    {
-      "epoch": 1.0,
-      "eval_loss": 8.250589370727539,
-      "eval_runtime": 2.0471,
-      "eval_samples_per_second": 4.885,
-      "eval_steps_per_second": 0.977,
-      "step": 13
-    },
-    {
-      "epoch": 1.1538461538461537,
-      "grad_norm": 15.25,
-      "learning_rate": 0.00019985583705641418,
-      "loss": 19.4813,
-      "step": 15
+      "epoch": 0.8571428571428571,
+      "eval_loss": 15.484689712524414,
+      "eval_runtime": 0.2675,
+      "eval_samples_per_second": 37.382,
+      "eval_steps_per_second": 3.738,
+      "step": 3
     },
     {
-      "epoch": 1.5384615384615383,
-      "grad_norm": 34.75,
-      "learning_rate": 0.00019823877374156647,
-      "loss": 16.6271,
-      "step": 20
-    },
-    {
-      "epoch": 1.9230769230769231,
-      "grad_norm": 21.875,
-      "learning_rate": 0.00019485364419471454,
-      "loss": 7.0034,
-      "step": 25
+      "epoch": 1.4285714285714286,
+      "grad_norm": 52.25,
+      "learning_rate": 0.00019730448705798239,
+      "loss": 38.5919,
+      "step": 5
     },
     {
       "epoch": 2.0,
-      "eval_loss": 3.5644798278808594,
-      "eval_runtime": 2.0643,
-      "eval_samples_per_second": 4.844,
-      "eval_steps_per_second": 0.969,
-      "step": 26
+      "eval_loss": 12.549888610839844,
+      "eval_runtime": 0.2336,
+      "eval_samples_per_second": 42.806,
+      "eval_steps_per_second": 4.281,
+      "step": 7
     },
     {
-      "epoch": 2.3076923076923075,
-      "grad_norm": 5.5625,
-      "learning_rate": 0.0001897613727639014,
-      "loss": 2.3031,
-      "step": 30
-    },
-    {
-      "epoch": 2.6923076923076925,
-      "grad_norm": 3.09375,
-      "learning_rate": 0.00018305360832480117,
-      "loss": 1.9278,
-      "step": 35
-    },
-    {
-      "epoch": 3.0,
-      "eval_loss": 2.7830893993377686,
-      "eval_runtime": 2.0487,
-      "eval_samples_per_second": 4.881,
-      "eval_steps_per_second": 0.976,
-      "step": 39
-    },
-    {
-      "epoch": 3.076923076923077,
-      "grad_norm": 1.8671875,
-      "learning_rate": 0.00017485107481711012,
-      "loss": 1.6366,
-      "step": 40
-    },
-    {
-      "epoch": 3.4615384615384617,
-      "grad_norm": 5.84375,
-      "learning_rate": 0.0001653013984983585,
-      "loss": 1.4332,
-      "step": 45
+      "epoch": 2.857142857142857,
+      "grad_norm": 11.375,
+      "learning_rate": 0.0001686241637868734,
+      "loss": 23.0632,
+      "step": 10
     },
     {
-      "epoch": 3.8461538461538463,
-      "grad_norm": 1.5703125,
-      "learning_rate": 0.00015457645101945046,
-      "loss": 1.3284,
-      "step": 50
+      "epoch": 2.857142857142857,
+      "eval_loss": 10.153787612915039,
+      "eval_runtime": 0.2414,
+      "eval_samples_per_second": 41.423,
+      "eval_steps_per_second": 4.142,
+      "step": 10
     },
     {
       "epoch": 4.0,
-      "eval_loss": 2.7295384407043457,
-      "eval_runtime": 2.0468,
-      "eval_samples_per_second": 4.886,
-      "eval_steps_per_second": 0.977,
-      "step": 52
-    },
-    {
-      "epoch": 4.230769230769231,
-      "grad_norm": 2.0625,
-      "learning_rate": 0.00014286925614030542,
-      "loss": 1.1714,
-      "step": 55
-    },
-    {
-      "epoch": 4.615384615384615,
-      "grad_norm": 1.7578125,
-      "learning_rate": 0.0001303905157574247,
-      "loss": 1.0989,
-      "step": 60
+      "eval_loss": 8.464659690856934,
+      "eval_runtime": 0.2419,
+      "eval_samples_per_second": 41.336,
+      "eval_steps_per_second": 4.134,
+      "step": 14
     },
     {
-      "epoch": 5.0,
-      "grad_norm": 2.921875,
+      "epoch": 4.285714285714286,
+      "grad_norm": 4.3125,
       "learning_rate": 0.00011736481776669306,
-      "loss": 1.046,
-      "step": 65
-    },
-    {
-      "epoch": 5.0,
-      "eval_loss": 2.7646915912628174,
-      "eval_runtime": 2.047,
-      "eval_samples_per_second": 4.885,
-      "eval_steps_per_second": 0.977,
-      "step": 65
+      "loss": 19.8584,
+      "step": 15
     },
     {
-      "epoch": 5.384615384615385,
-      "grad_norm": 2.65625,
-      "learning_rate": 0.00010402659401094152,
-      "loss": 0.9141,
-      "step": 70
+      "epoch": 4.857142857142857,
+      "eval_loss": 8.02161979675293,
+      "eval_runtime": 0.2493,
+      "eval_samples_per_second": 40.12,
+      "eval_steps_per_second": 4.012,
+      "step": 17
     },
     {
-      "epoch": 5.769230769230769,
-      "grad_norm": 1.7734375,
-      "learning_rate": 9.061590105968208e-05,
-      "loss": 0.8611,
-      "step": 75
+      "epoch": 5.714285714285714,
+      "grad_norm": 4.21875,
+      "learning_rate": 6.039202339608432e-05,
+      "loss": 19.1062,
+      "step": 20
     },
     {
       "epoch": 6.0,
-      "eval_loss": 2.862928867340088,
-      "eval_runtime": 2.0464,
-      "eval_samples_per_second": 4.887,
-      "eval_steps_per_second": 0.977,
-      "step": 78
-    },
-    {
-      "epoch": 6.153846153846154,
-      "grad_norm": 11.625,
-      "learning_rate": 7.73740997570278e-05,
-      "loss": 0.7981,
-      "step": 80
+      "eval_loss": 7.756901741027832,
+      "eval_runtime": 0.2345,
+      "eval_samples_per_second": 42.648,
+      "eval_steps_per_second": 4.265,
+      "step": 21
     },
     {
-      "epoch": 6.538461538461538,
-      "grad_norm": 2.09375,
-      "learning_rate": 6.453951129574644e-05,
-      "loss": 0.7228,
-      "step": 85
+      "epoch": 6.857142857142857,
+      "eval_loss": 7.677903175354004,
+      "eval_runtime": 0.25,
+      "eval_samples_per_second": 40.001,
+      "eval_steps_per_second": 4.0,
+      "step": 24
     },
     {
-      "epoch": 6.923076923076923,
-      "grad_norm": 2.078125,
-      "learning_rate": 5.234312799786921e-05,
-      "loss": 0.7242,
-      "step": 90
-    },
-    {
-      "epoch": 7.0,
-      "eval_loss": 3.040492296218872,
-      "eval_runtime": 2.0484,
-      "eval_samples_per_second": 4.882,
-      "eval_steps_per_second": 0.976,
-      "step": 91
-    },
-    {
-      "epoch": 7.3076923076923075,
-      "grad_norm": 1.421875,
-      "learning_rate": 4.100445599768774e-05,
-      "loss": 0.6348,
-      "step": 95
-    },
-    {
-      "epoch": 7.6923076923076925,
-      "grad_norm": 2.25,
-      "learning_rate": 3.072756464904006e-05,
-      "loss": 0.6192,
-      "step": 100
+      "epoch": 7.142857142857143,
+      "grad_norm": 4.46875,
+      "learning_rate": 1.6451218858706374e-05,
+      "loss": 18.5688,
+      "step": 25
     },
     {
       "epoch": 8.0,
-      "eval_loss": 3.1549479961395264,
-      "eval_runtime": 2.0433,
-      "eval_samples_per_second": 4.894,
-      "eval_steps_per_second": 0.979,
-      "step": 104
+      "eval_loss": 7.643762111663818,
+      "eval_runtime": 0.2433,
+      "eval_samples_per_second": 41.101,
+      "eval_steps_per_second": 4.11,
+      "step": 28
     },
     {
-      "epoch": 8.076923076923077,
-      "grad_norm": 1.3359375,
-      "learning_rate": 2.1697413758237784e-05,
-      "loss": 0.5742,
-      "step": 105
-    },
-    {
-      "epoch": 8.461538461538462,
-      "grad_norm": 1.5703125,
-      "learning_rate": 1.4076524743778319e-05,
-      "loss": 0.5514,
-      "step": 110
-    },
-    {
-      "epoch": 8.846153846153847,
-      "grad_norm": 1.40625,
-      "learning_rate": 8.002055634117578e-06,
-      "loss": 0.5312,
-      "step": 115
-    },
-    {
-      "epoch": 9.0,
-      "eval_loss": 3.3040664196014404,
-      "eval_runtime": 2.0464,
-      "eval_samples_per_second": 4.887,
-      "eval_steps_per_second": 0.977,
-      "step": 117
-    },
-    {
-      "epoch": 9.23076923076923,
-      "grad_norm": 1.265625,
-      "learning_rate": 3.5833325466437694e-06,
-      "loss": 0.5132,
-      "step": 120
-    },
-    {
-      "epoch": 9.615384615384615,
-      "grad_norm": 1.3984375,
-      "learning_rate": 8.998820754091531e-07,
-      "loss": 0.4966,
-      "step": 125
-    },
-    {
-      "epoch": 10.0,
-      "grad_norm": 1.2109375,
+      "epoch": 8.571428571428571,
+      "grad_norm": 4.84375,
       "learning_rate": 0.0,
-      "loss": 0.5202,
-      "step": 130
+      "loss": 18.5805,
+      "step": 30
     },
     {
-      "epoch": 10.0,
-      "eval_loss": 3.298276901245117,
-      "eval_runtime": 2.0482,
-      "eval_samples_per_second": 4.882,
-      "eval_steps_per_second": 0.976,
-      "step": 130
+      "epoch": 8.571428571428571,
+      "eval_loss": 7.6471662521362305,
+      "eval_runtime": 0.2333,
+      "eval_samples_per_second": 42.859,
+      "eval_steps_per_second": 4.286,
+      "step": 30
     },
     {
-      "epoch": 10.0,
-      "step": 130,
-      "total_flos": 9.937675599478784e+16,
-      "train_loss": 4.9734316147290745,
-      "train_runtime": 870.6795,
-      "train_samples_per_second": 2.366,
-      "train_steps_per_second": 0.149
+      "epoch": 8.571428571428571,
+      "step": 30,
+      "total_flos": 9.147533553041408e+16,
+      "train_loss": 23.272574106852215,
+      "train_runtime": 74.8732,
+      "train_samples_per_second": 29.383,
+      "train_steps_per_second": 0.401
     }
   ],
   "logging_steps": 5,
-  "max_steps": 130,
+  "max_steps": 30,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
   "save_steps": 100,
-  "total_flos": 9.937675599478784e+16,
+  "total_flos": 9.147533553041408e+16,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78b40d7c73ba44ff75f030f6e20bced997a2f0021ad1be42435051086b4f8e39
+oid sha256:4101eb7a5410558abced50ecc197ea8663a854eae1e0a5e72754ec45d82afdaa
 size 5176