Updated model

README.md

@@ -40,7 +40,7 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 2
-- training_steps: 6
+- training_steps: 200
 
 ### Training results
 

adapter_config.json

@@ -1,6 +1,6 @@
 {
   "auto_mapping": null,
-  "base_model_name_or_path": null,
+  "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -14,13 +14,13 @@
   "r": 16,
   "revision": null,
   "target_modules": [
+    "up_proj",
+    "v_proj",
+    "down_proj",
     "gate_proj",
     "o_proj",
-    "down_proj",
-    "k_proj",
     "q_proj",
-    "v_proj",
-    "up_proj"
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a2fc323a0408cee7d756cfb402c31c80a8dcd3e782dc114de20c42a63ed16a2
-size 160077005
+oid sha256:4c0fb3ea95a6c5cb64da617c99063dc56a3402f5b1b8fa595b086528289058db
+size 160069389

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 0.0,
-    "total_flos": 42650566778880.0,
-    "train_loss": 2.389628251393636,
-    "train_runtime": 39.2629,
-    "train_samples_per_second": 0.611,
-    "train_steps_per_second": 0.153
+    "epoch": 0.05,
+    "total_flos": 1428558744158208.0,
+    "train_loss": 1.3583835124969483,
+    "train_runtime": 1540.4432,
+    "train_samples_per_second": 0.519,
+    "train_steps_per_second": 0.13
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 0.0,
-    "total_flos": 42650566778880.0,
-    "train_loss": 2.389628251393636,
-    "train_runtime": 39.2629,
-    "train_samples_per_second": 0.611,
-    "train_steps_per_second": 0.153
+    "epoch": 0.05,
+    "total_flos": 1428558744158208.0,
+    "train_loss": 1.3583835124969483,
+    "train_runtime": 1540.4432,
+    "train_samples_per_second": 0.519,
+    "train_steps_per_second": 0.13
 }

trainer_state.json

@@ -1,28 +1,148 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.0013729977116704805,
+  "epoch": 0.04576659038901602,
   "eval_steps": 500,
-  "global_step": 6,
+  "global_step": 200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "step": 6,
-      "total_flos": 42650566778880.0,
-      "train_loss": 2.389628251393636,
-      "train_runtime": 39.2629,
-      "train_samples_per_second": 0.611,
-      "train_steps_per_second": 0.153
+      "learning_rate": 0.00019292929292929293,
+      "loss": 2.0259,
+      "step": 10
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018282828282828283,
+      "loss": 1.4961,
+      "step": 20
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00017272727272727275,
+      "loss": 1.3953,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00016262626262626264,
+      "loss": 1.3234,
+      "step": 40
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00015252525252525253,
+      "loss": 1.4601,
+      "step": 50
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00014242424242424243,
+      "loss": 1.3362,
+      "step": 60
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00013232323232323235,
+      "loss": 1.3905,
+      "step": 70
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00012222222222222224,
+      "loss": 1.3132,
+      "step": 80
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00011212121212121212,
+      "loss": 1.2767,
+      "step": 90
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00010202020202020202,
+      "loss": 1.3303,
+      "step": 100
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.191919191919192e-05,
+      "loss": 1.3685,
+      "step": 110
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 8.181818181818183e-05,
+      "loss": 1.335,
+      "step": 120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 7.171717171717171e-05,
+      "loss": 1.3122,
+      "step": 130
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 6.161616161616162e-05,
+      "loss": 1.2693,
+      "step": 140
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 5.151515151515152e-05,
+      "loss": 1.2069,
+      "step": 150
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 4.141414141414142e-05,
+      "loss": 1.2302,
+      "step": 160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 3.131313131313132e-05,
+      "loss": 1.2451,
+      "step": 170
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.1212121212121215e-05,
+      "loss": 1.2993,
+      "step": 180
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.1111111111111112e-05,
+      "loss": 1.2968,
+      "step": 190
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.0101010101010103e-06,
+      "loss": 1.2567,
+      "step": 200
+    },
+    {
+      "epoch": 0.05,
+      "step": 200,
+      "total_flos": 1428558744158208.0,
+      "train_loss": 1.3583835124969483,
+      "train_runtime": 1540.4432,
+      "train_samples_per_second": 0.519,
+      "train_steps_per_second": 0.13
     }
   ],
   "logging_steps": 10,
-  "max_steps": 6,
+  "max_steps": 200,
   "num_train_epochs": 1,
   "save_steps": 500,
-  "total_flos": 42650566778880.0,
+  "total_flos": 1428558744158208.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0ac0db2deb2139cb448a023a3dc53a82c952d78c89d126c7a619268f1a86d61
+oid sha256:6af3d684fb9b885bdc10d2bedf59709dfd4ef3c01f7e374a9a07ad916540d937
 size 4091