Commit
•
85245ed
1
Parent(s):
4affbd4
fine-tuning-Phi2-with-webglm-qa-with-lora_4
Browse files- README.md +51 -51
- adapter_config.json +6 -4
- adapter_model.safetensors +2 -2
README.md
CHANGED
|
@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
|
|
| 16 |
|
| 17 |
This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on an unknown dataset.
|
| 18 |
It achieves the following results on the evaluation set:
|
| 19 |
-
- Loss: 0.
|
| 20 |
|
| 21 |
## Model description
|
| 22 |
|
|
@@ -51,56 +51,56 @@ The following hyperparameters were used during training:
|
|
| 51 |
|
| 52 |
| Training Loss | Epoch | Step | Validation Loss |
|
| 53 |
|:-------------:|:-----:|:----:|:---------------:|
|
| 54 |
-
| 8.
|
| 55 |
-
|
|
| 56 |
-
|
|
| 57 |
-
|
|
| 58 |
-
|
|
| 59 |
-
|
|
| 60 |
-
|
|
| 61 |
-
| 0.
|
| 62 |
-
| 0.
|
| 63 |
-
| 0.
|
| 64 |
-
| 0.
|
| 65 |
-
| 0.
|
| 66 |
-
| 0.
|
| 67 |
-
| 0.
|
| 68 |
-
| 0.
|
| 69 |
-
| 0.
|
| 70 |
-
| 0.
|
| 71 |
-
| 0.
|
| 72 |
-
| 0.
|
| 73 |
-
| 0.
|
| 74 |
-
| 0.
|
| 75 |
-
| 0.
|
| 76 |
-
| 0.
|
| 77 |
-
| 0.
|
| 78 |
-
| 0.
|
| 79 |
-
| 0.
|
| 80 |
-
| 0.
|
| 81 |
-
| 0.
|
| 82 |
-
| 0.
|
| 83 |
-
| 0.
|
| 84 |
-
| 0.
|
| 85 |
-
| 0.
|
| 86 |
-
| 0.
|
| 87 |
-
| 0.
|
| 88 |
-
| 0.
|
| 89 |
-
| 0.
|
| 90 |
-
| 0.
|
| 91 |
-
| 0.
|
| 92 |
-
| 0.
|
| 93 |
-
| 0.
|
| 94 |
-
| 0.
|
| 95 |
-
| 0.
|
| 96 |
-
| 0.
|
| 97 |
-
| 0.
|
| 98 |
-
| 0.
|
| 99 |
-
| 0.
|
| 100 |
-
| 0.
|
| 101 |
-
| 0.
|
| 102 |
-
| 0.
|
| 103 |
-
| 0.
|
| 104 |
|
| 105 |
|
| 106 |
### Framework versions
|
|
|
|
| 16 |
|
| 17 |
This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on an unknown dataset.
|
| 18 |
It achieves the following results on the evaluation set:
|
| 19 |
+
- Loss: 0.1176
|
| 20 |
|
| 21 |
## Model description
|
| 22 |
|
|
|
|
| 51 |
|
| 52 |
| Training Loss | Epoch | Step | Validation Loss |
|
| 53 |
|:-------------:|:-----:|:----:|:---------------:|
|
| 54 |
+
| 8.1178 | 0.2 | 10 | 7.7550 |
|
| 55 |
+
| 7.3762 | 0.4 | 20 | 6.3827 |
|
| 56 |
+
| 4.9217 | 0.6 | 30 | 3.2172 |
|
| 57 |
+
| 1.7792 | 0.8 | 40 | 0.6700 |
|
| 58 |
+
| 0.5779 | 1.0 | 50 | 0.5969 |
|
| 59 |
+
| 0.4824 | 1.2 | 60 | 0.5149 |
|
| 60 |
+
| 0.4689 | 1.39 | 70 | 0.4440 |
|
| 61 |
+
| 0.3833 | 1.59 | 80 | 0.3862 |
|
| 62 |
+
| 0.2916 | 1.79 | 90 | 0.3364 |
|
| 63 |
+
| 0.2435 | 1.99 | 100 | 0.3013 |
|
| 64 |
+
| 0.2538 | 2.19 | 110 | 0.2779 |
|
| 65 |
+
| 0.2147 | 2.39 | 120 | 0.2619 |
|
| 66 |
+
| 0.1982 | 2.59 | 130 | 0.2453 |
|
| 67 |
+
| 0.2183 | 2.79 | 140 | 0.2275 |
|
| 68 |
+
| 0.1737 | 2.99 | 150 | 0.2148 |
|
| 69 |
+
| 0.1794 | 3.19 | 160 | 0.2068 |
|
| 70 |
+
| 0.1692 | 3.39 | 170 | 0.1949 |
|
| 71 |
+
| 0.1573 | 3.59 | 180 | 0.1864 |
|
| 72 |
+
| 0.1478 | 3.78 | 190 | 0.1788 |
|
| 73 |
+
| 0.164 | 3.98 | 200 | 0.1732 |
|
| 74 |
+
| 0.1454 | 4.18 | 210 | 0.1676 |
|
| 75 |
+
| 0.1279 | 4.38 | 220 | 0.1653 |
|
| 76 |
+
| 0.1544 | 4.58 | 230 | 0.1595 |
|
| 77 |
+
| 0.1206 | 4.78 | 240 | 0.1524 |
|
| 78 |
+
| 0.1334 | 4.98 | 250 | 0.1486 |
|
| 79 |
+
| 0.1342 | 5.18 | 260 | 0.1472 |
|
| 80 |
+
| 0.1061 | 5.38 | 270 | 0.1442 |
|
| 81 |
+
| 0.1265 | 5.58 | 280 | 0.1427 |
|
| 82 |
+
| 0.131 | 5.78 | 290 | 0.1389 |
|
| 83 |
+
| 0.1067 | 5.98 | 300 | 0.1374 |
|
| 84 |
+
| 0.1158 | 6.18 | 310 | 0.1331 |
|
| 85 |
+
| 0.1114 | 6.37 | 320 | 0.1323 |
|
| 86 |
+
| 0.1104 | 6.57 | 330 | 0.1311 |
|
| 87 |
+
| 0.108 | 6.77 | 340 | 0.1281 |
|
| 88 |
+
| 0.1015 | 6.97 | 350 | 0.1271 |
|
| 89 |
+
| 0.1 | 7.17 | 360 | 0.1262 |
|
| 90 |
+
| 0.1091 | 7.37 | 370 | 0.1242 |
|
| 91 |
+
| 0.1013 | 7.57 | 380 | 0.1230 |
|
| 92 |
+
| 0.1074 | 7.77 | 390 | 0.1233 |
|
| 93 |
+
| 0.0946 | 7.97 | 400 | 0.1226 |
|
| 94 |
+
| 0.0854 | 8.17 | 410 | 0.1222 |
|
| 95 |
+
| 0.0914 | 8.37 | 420 | 0.1205 |
|
| 96 |
+
| 0.1117 | 8.57 | 430 | 0.1198 |
|
| 97 |
+
| 0.0922 | 8.76 | 440 | 0.1194 |
|
| 98 |
+
| 0.1012 | 8.96 | 450 | 0.1185 |
|
| 99 |
+
| 0.0964 | 9.16 | 460 | 0.1185 |
|
| 100 |
+
| 0.0948 | 9.36 | 470 | 0.1181 |
|
| 101 |
+
| 0.0943 | 9.56 | 480 | 0.1178 |
|
| 102 |
+
| 0.0915 | 9.76 | 490 | 0.1176 |
|
| 103 |
+
| 0.0924 | 9.96 | 500 | 0.1176 |
|
| 104 |
|
| 105 |
|
| 106 |
### Framework versions
|
adapter_config.json
CHANGED
|
@@ -9,20 +9,22 @@
|
|
| 9 |
"layers_pattern": null,
|
| 10 |
"layers_to_transform": null,
|
| 11 |
"loftq_config": {},
|
| 12 |
-
"lora_alpha":
|
| 13 |
"lora_dropout": 0.05,
|
| 14 |
"megatron_config": null,
|
| 15 |
"megatron_core": "megatron.core",
|
| 16 |
"modules_to_save": null,
|
| 17 |
"peft_type": "LORA",
|
| 18 |
-
"r":
|
| 19 |
"rank_pattern": {},
|
| 20 |
"revision": null,
|
| 21 |
"target_modules": [
|
| 22 |
"q_proj",
|
|
|
|
|
|
|
|
|
|
| 23 |
"v_proj",
|
| 24 |
-
"
|
| 25 |
-
"k_proj"
|
| 26 |
],
|
| 27 |
"task_type": "CAUSAL_LM"
|
| 28 |
}
|
|
|
|
| 9 |
"layers_pattern": null,
|
| 10 |
"layers_to_transform": null,
|
| 11 |
"loftq_config": {},
|
| 12 |
+
"lora_alpha": 32,
|
| 13 |
"lora_dropout": 0.05,
|
| 14 |
"megatron_config": null,
|
| 15 |
"megatron_core": "megatron.core",
|
| 16 |
"modules_to_save": null,
|
| 17 |
"peft_type": "LORA",
|
| 18 |
+
"r": 16,
|
| 19 |
"rank_pattern": {},
|
| 20 |
"revision": null,
|
| 21 |
"target_modules": [
|
| 22 |
"q_proj",
|
| 23 |
+
"fc1",
|
| 24 |
+
"fc2",
|
| 25 |
+
"k_proj",
|
| 26 |
"v_proj",
|
| 27 |
+
"dense"
|
|
|
|
| 28 |
],
|
| 29 |
"task_type": "CAUSAL_LM"
|
| 30 |
}
|
adapter_model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e662e15db52fcb238ea6c11dbfbd4e19c2db88459a035c5c402aa17514af1685
|
| 3 |
+
size 94422368
|