Model save

README.md

@@ -20,7 +20,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 14.0161
+- Loss: 1.7210
 
 ## Model description
 
@@ -57,12 +57,12 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 13.938        | 0.9945 | 91   | 14.0161         |
+| 1.7241        | 0.9945 | 91   | 1.7210          |
 
 
 ### Framework versions
 
-- PEFT 0.13.3.dev0
+- PEFT 0.13.2
 - Transformers 4.46.2
 - Pytorch 2.3.1+cu121
 - Datasets 3.1.0

all_results.json

@@ -1,14 +1,9 @@
 {
     "epoch": 0.994535519125683,
-    "eval_loss": 14.016096115112305,
-    "eval_runtime": 2.0922,
-    "eval_samples": 518,
-    "eval_samples_per_second": 83.167,
-    "eval_steps_per_second": 0.956,
     "total_flos": 4.03002565094015e+17,
-    "train_loss": 13.96694589971186,
-    "train_runtime": 1243.0718,
+    "train_loss": 1.799644404715234,
+    "train_runtime": 332.7348,
     "train_samples": 51241,
-    "train_samples_per_second": 14.102,
-    "train_steps_per_second": 0.073
+    "train_samples_per_second": 52.685,
+    "train_steps_per_second": 0.273
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.994535519125683,
     "total_flos": 4.03002565094015e+17,
-    "train_loss": 13.96694589971186,
-    "train_runtime": 1243.0718,
+    "train_loss": 1.799644404715234,
+    "train_runtime": 332.7348,
     "train_samples": 51241,
-    "train_samples_per_second": 14.102,
-    "train_steps_per_second": 0.073
+    "train_samples_per_second": 52.685,
+    "train_steps_per_second": 0.273
 }

trainer_state.json

@@ -10,153 +10,153 @@
   "log_history": [
     {
       "epoch": 0.01092896174863388,
-      "grad_norm": 1.7711571016959918e+18,
+      "grad_norm": 67.9917221069336,
       "learning_rate": 2e-05,
-      "loss": 14.0238,
+      "loss": 2.2916,
       "step": 1
     },
     {
       "epoch": 0.0546448087431694,
-      "grad_norm": 3.320609228523045e+18,
+      "grad_norm": 2.097611665725708,
       "learning_rate": 0.0001,
-      "loss": 13.9905,
+      "loss": 2.2127,
       "step": 5
     },
     {
       "epoch": 0.1092896174863388,
-      "grad_norm": 1.1468547292982673e+19,
+      "grad_norm": 1.9431463479995728,
       "learning_rate": 0.0002,
-      "loss": 14.0267,
+      "loss": 2.0887,
       "step": 10
     },
     {
       "epoch": 0.16393442622950818,
-      "grad_norm": 3.9691701809399726e+18,
+      "grad_norm": 1.8884824514389038,
       "learning_rate": 0.00019812553106273847,
-      "loss": 14.0288,
+      "loss": 1.9685,
       "step": 15
     },
     {
       "epoch": 0.2185792349726776,
-      "grad_norm": 1.0431252630176006e+19,
+      "grad_norm": 2.330018997192383,
       "learning_rate": 0.00019257239692688907,
-      "loss": 14.0141,
+      "loss": 1.8856,
       "step": 20
     },
     {
       "epoch": 0.273224043715847,
-      "grad_norm": 1.0054431303639499e+19,
+      "grad_norm": 0.6340972781181335,
       "learning_rate": 0.00018354878114129367,
-      "loss": 13.9837,
+      "loss": 1.8482,
       "step": 25
     },
     {
       "epoch": 0.32786885245901637,
-      "grad_norm": 4.974251626953966e+18,
+      "grad_norm": 0.45715248584747314,
       "learning_rate": 0.00017139297345578994,
-      "loss": 13.9929,
+      "loss": 1.7705,
       "step": 30
     },
     {
       "epoch": 0.3825136612021858,
-      "grad_norm": Infinity,
+      "grad_norm": 1.0445672273635864,
       "learning_rate": 0.00015656068754865387,
-      "loss": 13.9978,
+      "loss": 1.7627,
       "step": 35
     },
     {
       "epoch": 0.4371584699453552,
-      "grad_norm": Infinity,
+      "grad_norm": 0.503304123878479,
       "learning_rate": 0.0001396079766039157,
-      "loss": 13.9489,
+      "loss": 1.7488,
       "step": 40
     },
     {
       "epoch": 0.4918032786885246,
-      "grad_norm": 1.6553702809539707e+19,
+      "grad_norm": 0.5089389681816101,
       "learning_rate": 0.0001211703872229411,
-      "loss": 13.9487,
+      "loss": 1.7165,
       "step": 45
     },
     {
       "epoch": 0.546448087431694,
-      "grad_norm": Infinity,
+      "grad_norm": 0.416858971118927,
       "learning_rate": 0.00010193913317718244,
-      "loss": 13.9381,
+      "loss": 1.7396,
       "step": 50
     },
     {
       "epoch": 0.6010928961748634,
-      "grad_norm": 4.656146420363952e+18,
+      "grad_norm": 0.29764389991760254,
       "learning_rate": 8.263518223330697e-05,
-      "loss": 13.9454,
+      "loss": 1.7045,
       "step": 55
     },
     {
       "epoch": 0.6557377049180327,
-      "grad_norm": 8.26754403884073e+18,
+      "grad_norm": 0.3781444728374481,
       "learning_rate": 6.398222751952899e-05,
-      "loss": 13.9116,
+      "loss": 1.6977,
       "step": 60
     },
     {
       "epoch": 0.7103825136612022,
-      "grad_norm": Infinity,
+      "grad_norm": 0.3005172610282898,
       "learning_rate": 4.66795567198309e-05,
-      "loss": 13.9312,
+      "loss": 1.715,
       "step": 65
     },
     {
       "epoch": 0.7650273224043715,
-      "grad_norm": 3.3132386523262484e+18,
+      "grad_norm": 0.39595624804496765,
       "learning_rate": 3.137583621312665e-05,
-      "loss": 13.9726,
+      "loss": 1.7143,
       "step": 70
     },
     {
       "epoch": 0.819672131147541,
-      "grad_norm": 3.950367432593375e+18,
+      "grad_norm": 0.4393070638179779,
       "learning_rate": 1.864479297370325e-05,
-      "loss": 13.9497,
+      "loss": 1.6919,
       "step": 75
     },
     {
       "epoch": 0.8743169398907104,
-      "grad_norm": 3.439258352910074e+18,
+      "grad_norm": 0.38544324040412903,
       "learning_rate": 8.963705903385345e-06,
-      "loss": 13.9259,
+      "loss": 1.7241,
       "step": 80
     },
     {
       "epoch": 0.9289617486338798,
-      "grad_norm": Infinity,
+      "grad_norm": 0.5594367980957031,
       "learning_rate": 2.6955129420176196e-06,
-      "loss": 13.9617,
+      "loss": 1.6885,
       "step": 85
     },
     {
       "epoch": 0.9836065573770492,
-      "grad_norm": Infinity,
+      "grad_norm": 0.2916119396686554,
       "learning_rate": 7.520474957699586e-08,
-      "loss": 13.938,
+      "loss": 1.7241,
       "step": 90
     },
     {
       "epoch": 0.994535519125683,
-      "eval_loss": 14.016096115112305,
-      "eval_runtime": 2.0876,
-      "eval_samples_per_second": 83.349,
-      "eval_steps_per_second": 0.958,
+      "eval_loss": 1.7210174798965454,
+      "eval_runtime": 1.2379,
+      "eval_samples_per_second": 140.557,
+      "eval_steps_per_second": 1.616,
       "step": 91
     },
     {
       "epoch": 0.994535519125683,
       "step": 91,
       "total_flos": 4.03002565094015e+17,
-      "train_loss": 13.96694589971186,
-      "train_runtime": 1243.0718,
-      "train_samples_per_second": 14.102,
-      "train_steps_per_second": 0.073
+      "train_loss": 1.799644404715234,
+      "train_runtime": 332.7348,
+      "train_samples_per_second": 52.685,
+      "train_steps_per_second": 0.273
     }
   ],
   "logging_steps": 5,