Model save

README.md

@@ -4,18 +4,18 @@ base_model: 01-ai/Yi-6B
 tags:
 - generated_from_trainer
 model-index:
-- name: apt-chat-yi-34b-sft-full
+- name: apt-chat-yi-6B-sft-full
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# apt-chat-yi-34b-sft-full
+# apt-chat-yi-6B-sft-full
 
 This model is a fine-tuned version of [01-ai/Yi-6B](https://huggingface.co/01-ai/Yi-6B) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.3422
+- Loss: 1.0677
 
 ## Model description
 
@@ -39,26 +39,25 @@ The following hyperparameters were used during training:
 - eval_batch_size: 1
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 6
+- num_devices: 8
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 24
-- total_eval_batch_size: 6
+- total_train_batch_size: 32
+- total_eval_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
-- num_epochs: 3
+- num_epochs: 2
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.1029        | 0.15  | 135  | 1.0746          |
-| 0.8271        | 1.15  | 270  | 1.1266          |
-| 0.5131        | 2.15  | 405  | 1.3422          |
+| 1.0548        | 0.15  | 1368 | 1.0247          |
+| 0.9254        | 1.15  | 2736 | 1.0677          |
 
 
 ### Framework versions
 
 - Transformers 4.35.0
-- Pytorch 2.1.0.dev20230605+cu121
+- Pytorch 2.1.0+cu118
 - Datasets 2.14.6
 - Tokenizers 0.14.1

all_results.json

@@ -1,13 +1,13 @@
 {
-    "epoch": 2.15,
-    "eval_loss": 1.3422094583511353,
-    "eval_runtime": 4.6035,
+    "epoch": 1.15,
+    "eval_loss": 1.0676991939544678,
+    "eval_runtime": 4.4863,
     "eval_samples": 500,
-    "eval_samples_per_second": 108.613,
-    "eval_steps_per_second": 18.247,
-    "train_loss": 0.8341983215308484,
-    "train_runtime": 1136.3762,
-    "train_samples": 21407,
-    "train_samples_per_second": 56.514,
-    "train_steps_per_second": 2.355
+    "eval_samples_per_second": 111.451,
+    "eval_steps_per_second": 14.043,
+    "train_loss": 0.9719247023264567,
+    "train_runtime": 13352.0365,
+    "train_samples": 285436,
+    "train_samples_per_second": 42.755,
+    "train_steps_per_second": 1.336
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.15,
-    "eval_loss": 1.3422094583511353,
-    "eval_runtime": 4.6035,
+    "epoch": 1.15,
+    "eval_loss": 1.0676991939544678,
+    "eval_runtime": 4.4863,
     "eval_samples": 500,
-    "eval_samples_per_second": 108.613,
-    "eval_steps_per_second": 18.247
+    "eval_samples_per_second": 111.451,
+    "eval_steps_per_second": 14.043
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:38adde306149008ebde6bd1c47df7c569461f342c6e44c5662babc9a9aa8fcba
+oid sha256:374370d80ea8bab03b1355f3dbd5b5dbd44c7043e0798d6f44b6dc4c9c18a45b
 size 4932711032

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae8166c4e72f2bb5633bbea473189925f9f8190892eedebee22ed7f4a13da880
+oid sha256:7645743c9ee76cd989bc5ec83312837f22fb36dd590496f415fb778ddd1d8707
 size 4976802304

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8c3783f503806d1a8e7a383780717e1c9c2772e63f37f86e4423c6c8bda6cd4
+oid sha256:2dc189809d22dc30a9a7d52f17b178e73925e2e40e63d22c874158e212f2a6ab
 size 2212590400

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.15,
-    "train_loss": 0.8341983215308484,
-    "train_runtime": 1136.3762,
-    "train_samples": 21407,
-    "train_samples_per_second": 56.514,
-    "train_steps_per_second": 2.355
+    "epoch": 1.15,
+    "train_loss": 0.9719247023264567,
+    "train_runtime": 13352.0365,
+    "train_samples": 285436,
+    "train_samples_per_second": 42.755,
+    "train_steps_per_second": 1.336
 }

trainer_state.json

@@ -1,106 +1,374 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.151345291479821,
+  "epoch": 1.1533632286995517,
   "eval_steps": 500,
-  "global_step": 405,
+  "global_step": 2736,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 1.9999993108758315e-05,
-      "loss": 1.5531,
+      "learning_rate": 1.9999999844947046e-05,
+      "loss": 1.7024,
       "step": 1
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 1.9982776840027333e-05,
-      "loss": 1.177,
+      "epoch": 0.01,
+      "learning_rate": 1.999961237011484e-05,
+      "loss": 1.1507,
       "step": 50
     },
     {
-      "epoch": 0.11,
-      "learning_rate": 1.993116668755721e-05,
-      "loss": 1.1029,
+      "epoch": 0.01,
+      "learning_rate": 1.9998449510510744e-05,
+      "loss": 1.0928,
       "step": 100
     },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.999651151133954e-05,
+      "loss": 1.0793,
+      "step": 150
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.999379852284651e-05,
+      "loss": 1.0867,
+      "step": 200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.999031075535873e-05,
+      "loss": 1.0857,
+      "step": 250
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.9986048479268788e-05,
+      "loss": 1.0721,
+      "step": 300
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.99810120250138e-05,
+      "loss": 1.0923,
+      "step": 350
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9975201783049804e-05,
+      "loss": 1.0836,
+      "step": 400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.9968618203821487e-05,
+      "loss": 1.0769,
+      "step": 450
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.9961261797727256e-05,
+      "loss": 1.0574,
+      "step": 500
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.9953133135079686e-05,
+      "loss": 1.042,
+      "step": 550
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.9944232846061284e-05,
+      "loss": 1.0554,
+      "step": 600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.993456162067566e-05,
+      "loss": 1.0735,
+      "step": 650
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.992412020869401e-05,
+      "loss": 1.0785,
+      "step": 700
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.9912909419596993e-05,
+      "loss": 1.0654,
+      "step": 750
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.9900930122511993e-05,
+      "loss": 1.0606,
+      "step": 800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.988818324614572e-05,
+      "loss": 1.0664,
+      "step": 850
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.9874669778712215e-05,
+      "loss": 1.0604,
+      "step": 900
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9860390767856244e-05,
+      "loss": 1.0674,
+      "step": 950
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.984534732057208e-05,
+      "loss": 1.042,
+      "step": 1000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9829540603117667e-05,
+      "loss": 1.0452,
+      "step": 1050
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9812971840924222e-05,
+      "loss": 1.0577,
+      "step": 1100
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.979564231850122e-05,
+      "loss": 1.0471,
+      "step": 1150
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.977755337933682e-05,
+      "loss": 1.0704,
+      "step": 1200
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9758706425793702e-05,
+      "loss": 1.0282,
+      "step": 1250
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.973910291900036e-05,
+      "loss": 1.0515,
+      "step": 1300
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.97187443787378e-05,
+      "loss": 1.0548,
+      "step": 1350
+    },
     {
       "epoch": 0.15,
-      "eval_loss": 1.0746039152145386,
-      "eval_runtime": 3.9638,
-      "eval_samples_per_second": 126.14,
-      "eval_steps_per_second": 21.192,
-      "step": 135
+      "eval_loss": 1.0247304439544678,
+      "eval_runtime": 4.5889,
+      "eval_samples_per_second": 108.959,
+      "eval_steps_per_second": 13.729,
+      "step": 1368
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.9697632383321755e-05,
+      "loss": 0.9636,
+      "step": 1400
+    },
+    {
+      "epoch": 1.01,
+      "learning_rate": 1.96757685694803e-05,
+      "loss": 0.9026,
+      "step": 1450
+    },
+    {
+      "epoch": 1.01,
+      "learning_rate": 1.965315463222695e-05,
+      "loss": 0.8808,
+      "step": 1500
     },
     {
       "epoch": 1.02,
-      "learning_rate": 1.984534732057208e-05,
-      "loss": 1.036,
-      "step": 150
+      "learning_rate": 1.9629792324729302e-05,
+      "loss": 0.8712,
+      "step": 1550
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 1.960568345817306e-05,
+      "loss": 0.8967,
+      "step": 1600
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 1.9580829901621666e-05,
+      "loss": 0.8676,
+      "step": 1650
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 1.9555233581871366e-05,
+      "loss": 0.8723,
+      "step": 1700
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 1.9528896483301866e-05,
+      "loss": 0.9122,
+      "step": 1750
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 1.9501820647722458e-05,
+      "loss": 0.8687,
+      "step": 1800
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 1.947400817421375e-05,
+      "loss": 0.8726,
+      "step": 1850
+    },
+    {
+      "epoch": 1.06,
+      "learning_rate": 1.944546121896493e-05,
+      "loss": 0.8505,
+      "step": 1900
     },
     {
       "epoch": 1.07,
-      "learning_rate": 1.9725614355209207e-05,
-      "loss": 0.8394,
-      "step": 200
+      "learning_rate": 1.9416181995106585e-05,
+      "loss": 0.8458,
+      "step": 1950
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 1.9386172772539162e-05,
+      "loss": 0.8721,
+      "step": 2000
+    },
+    {
+      "epoch": 1.08,
+      "learning_rate": 1.9355435877756957e-05,
+      "loss": 0.8676,
+      "step": 2050
+    },
+    {
+      "epoch": 1.08,
+      "learning_rate": 1.9323973693667762e-05,
+      "loss": 0.8826,
+      "step": 2100
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 1.929178865940815e-05,
+      "loss": 0.8607,
+      "step": 2150
+    },
+    {
+      "epoch": 1.09,
+      "learning_rate": 1.925888327015434e-05,
+      "loss": 0.8561,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 1.9225260076928783e-05,
+      "loss": 0.8687,
+      "step": 2250
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 1.919092168640239e-05,
+      "loss": 0.874,
+      "step": 2300
+    },
+    {
+      "epoch": 1.11,
+      "learning_rate": 1.915587076069243e-05,
+      "loss": 0.8563,
+      "step": 2350
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 1.9120110017156172e-05,
+      "loss": 0.8445,
+      "step": 2400
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 1.908364222818019e-05,
+      "loss": 0.8646,
+      "step": 2450
     },
     {
       "epoch": 1.13,
-      "learning_rate": 1.957238022747188e-05,
-      "loss": 0.8271,
-      "step": 250
+      "learning_rate": 1.9046470220965457e-05,
+      "loss": 0.8479,
+      "step": 2500
     },
     {
-      "epoch": 1.15,
-      "eval_loss": 1.1265954971313477,
-      "eval_runtime": 3.6594,
-      "eval_samples_per_second": 136.636,
-      "eval_steps_per_second": 22.955,
-      "step": 270
+      "epoch": 1.13,
+      "learning_rate": 1.9008596877308157e-05,
+      "loss": 0.8788,
+      "step": 2550
     },
     {
-      "epoch": 2.03,
-      "learning_rate": 1.9386172772539162e-05,
-      "loss": 0.667,
-      "step": 300
+      "epoch": 1.14,
+      "learning_rate": 1.8970025133376252e-05,
+      "loss": 0.9,
+      "step": 2600
     },
     {
-      "epoch": 2.09,
-      "learning_rate": 1.916763340656793e-05,
-      "loss": 0.5326,
-      "step": 350
+      "epoch": 1.14,
+      "learning_rate": 1.893075797948188e-05,
+      "loss": 0.8791,
+      "step": 2650
     },
     {
-      "epoch": 2.15,
-      "learning_rate": 1.8917514917250276e-05,
-      "loss": 0.5131,
-      "step": 400
+      "epoch": 1.15,
+      "learning_rate": 1.889079845984951e-05,
+      "loss": 0.9254,
+      "step": 2700
     },
     {
-      "epoch": 2.15,
-      "eval_loss": 1.3422094583511353,
-      "eval_runtime": 4.9335,
-      "eval_samples_per_second": 101.349,
-      "eval_steps_per_second": 17.027,
-      "step": 405
+      "epoch": 1.15,
+      "eval_loss": 1.0676991939544678,
+      "eval_runtime": 4.5191,
+      "eval_samples_per_second": 110.641,
+      "eval_steps_per_second": 13.941,
+      "step": 2736
     },
     {
-      "epoch": 2.15,
-      "step": 405,
-      "total_flos": 63559811727360.0,
-      "train_loss": 0.8341983215308484,
-      "train_runtime": 1136.3762,
-      "train_samples_per_second": 56.514,
-      "train_steps_per_second": 2.355
+      "epoch": 1.15,
+      "step": 2736,
+      "total_flos": 572810393026560.0,
+      "train_loss": 0.9719247023264567,
+      "train_runtime": 13352.0365,
+      "train_samples_per_second": 42.755,
+      "train_steps_per_second": 1.336
     }
   ],
   "logging_steps": 50,
-  "max_steps": 2676,
-  "num_train_epochs": 3,
+  "max_steps": 17840,
+  "num_train_epochs": 2,
   "save_steps": 500,
-  "total_flos": 63559811727360.0,
+  "total_flos": 572810393026560.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:38fa2e8b442d3c15b05cc3aad3c4f2d90f76d575a77420975deb6bc91e346800
+oid sha256:f08e31bceb220d21152bd52b1b3723abfb215236126fc2fed6cf222adf0775bd
 size 5624