Hugging Face's logo

dominic5
/
Project5_V3_Mistral8x7b_V2.2.5

PEFT
Safetensors
trl
sft
Generated from Trainer
Model card Files Community
dominic5 commited on
Commit
a114c2d
•
1 Parent(s): 7d5d296

Model save

Browse files
Files changed (10) hide show
  1. README.md +90 -0
  2. adapter_config.json +29 -0
  3. adapter_model.safetensors +3 -0
  4. all_results.json +9 -0
  5. special_tokens_map.json +24 -0
  6. tokenizer.json +0 -0
  7. tokenizer_config.json +43 -0
  8. train_results.json +9 -0
  9. trainer_state.json +294 -0
  10. training_args.bin +3 -0
README.md ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ library_name: peft
4
+ tags:
5
+ - trl
6
+ - sft
7
+ - generated_from_trainer
8
+ datasets:
9
+ - generator
10
+ base_model: mistralai/Mixtral-8x7B-v0.1
11
+ model-index:
12
+ - name: Project5_V3_Mistral8x7b_V2.2.5
13
+ results: []
14
+ ---
15
+
16
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
17
+ should probably proofread and complete it, then remove this comment. -->
18
+
19
+ # Project5_V3_Mistral8x7b_V2.2.5
20
+
21
+ This model is a fine-tuned version of [mistralai/Mixtral-8x7B-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-v0.1) on the generator dataset.
22
+ It achieves the following results on the evaluation set:
23
+ - Loss: 2.2430
24
+
25
+ ## Model description
26
+
27
+ More information needed
28
+
29
+ ## Intended uses & limitations
30
+
31
+ More information needed
32
+
33
+ ## Training and evaluation data
34
+
35
+ More information needed
36
+
37
+ ## Training procedure
38
+
39
+ ### Training hyperparameters
40
+
41
+ The following hyperparameters were used during training:
42
+ - learning_rate: 2e-05
43
+ - train_batch_size: 2
44
+ - eval_batch_size: 2
45
+ - seed: 42
46
+ - gradient_accumulation_steps: 128
47
+ - total_train_batch_size: 256
48
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
49
+ - lr_scheduler_type: cosine
50
+ - num_epochs: 40
51
+
52
+ ### Training results
53
+
54
+ | Training Loss | Epoch | Step | Validation Loss |
55
+ |:-------------:|:-----:|:----:|:---------------:|
56
+ | No log | 0.63 | 1 | 2.3024 |
57
+ | No log | 1.88 | 3 | 2.2997 |
58
+ | No log | 2.51 | 4 | 2.2977 |
59
+ | 2.2951 | 3.76 | 6 | 2.2935 |
60
+ | 2.2951 | 4.39 | 7 | 2.2905 |
61
+ | 2.2951 | 5.65 | 9 | 2.2853 |
62
+ | 2.2845 | 6.9 | 11 | 2.2789 |
63
+ | 2.2845 | 7.53 | 12 | 2.2757 |
64
+ | 2.2845 | 8.78 | 14 | 2.2696 |
65
+ | 2.2729 | 9.41 | 15 | 2.2667 |
66
+ | 2.2729 | 10.67 | 17 | 2.2619 |
67
+ | 2.2729 | 11.92 | 19 | 2.2576 |
68
+ | 2.2599 | 12.55 | 20 | 2.2555 |
69
+ | 2.2599 | 13.8 | 22 | 2.2519 |
70
+ | 2.2599 | 14.43 | 23 | 2.2504 |
71
+ | 2.2523 | 15.69 | 25 | 2.2479 |
72
+ | 2.2523 | 16.94 | 27 | 2.2462 |
73
+ | 2.2523 | 17.57 | 28 | 2.2454 |
74
+ | 2.2471 | 18.82 | 30 | 2.2442 |
75
+ | 2.2471 | 19.45 | 31 | 2.2437 |
76
+ | 2.2471 | 20.71 | 33 | 2.2432 |
77
+ | 2.2444 | 21.96 | 35 | 2.2427 |
78
+ | 2.2444 | 22.59 | 36 | 2.2428 |
79
+ | 2.2444 | 23.84 | 38 | 2.2429 |
80
+ | 2.2444 | 24.47 | 39 | 2.2429 |
81
+ | 2.2427 | 25.1 | 40 | 2.2430 |
82
+
83
+
84
+ ### Framework versions
85
+
86
+ - PEFT 0.8.2
87
+ - Transformers 4.38.1
88
+ - Pytorch 2.2.1+cu121
89
+ - Datasets 2.17.1
90
+ - Tokenizers 0.15.2
adapter_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "mistralai/Mixtral-8x7B-v0.1",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "loftq_config": {},
12
+ "lora_alpha": 16,
13
+ "lora_dropout": 0.1,
14
+ "megatron_config": null,
15
+ "megatron_core": "megatron.core",
16
+ "modules_to_save": null,
17
+ "peft_type": "LORA",
18
+ "r": 64,
19
+ "rank_pattern": {},
20
+ "revision": null,
21
+ "target_modules": [
22
+ "v_proj",
23
+ "o_proj",
24
+ "q_proj",
25
+ "k_proj"
26
+ ],
27
+ "task_type": "CAUSAL_LM",
28
+ "use_rslora": false
29
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6dab35bab81cca524c5ca8f5e2306894403dcf7742ac037232fd377bbc78a0a8
3
+ size 109086672
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 25.1,
3
+ "total_flos": 5.866939585627423e+18,
4
+ "train_loss": 2.2623767852783203,
5
+ "train_runtime": 20272.0714,
6
+ "train_samples": 73,
7
+ "train_samples_per_second": 0.805,
8
+ "train_steps_per_second": 0.002
9
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ }
29
+ },
30
+ "additional_special_tokens": [],
31
+ "bos_token": "<s>",
32
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
33
+ "clean_up_tokenization_spaces": false,
34
+ "eos_token": "</s>",
35
+ "legacy": true,
36
+ "model_max_length": 2048,
37
+ "pad_token": "</s>",
38
+ "sp_model_kwargs": {},
39
+ "spaces_between_special_tokens": false,
40
+ "tokenizer_class": "LlamaTokenizer",
41
+ "unk_token": "<unk>",
42
+ "use_default_system_prompt": false
43
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 25.1,
3
+ "total_flos": 5.866939585627423e+18,
4
+ "train_loss": 2.2623767852783203,
5
+ "train_runtime": 20272.0714,
6
+ "train_samples": 73,
7
+ "train_samples_per_second": 0.805,
8
+ "train_steps_per_second": 0.002
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 25.098039215686274,
5
+ "eval_steps": 500,
6
+ "global_step": 40,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.63,
13
+ "eval_loss": 2.3024485111236572,
14
+ "eval_runtime": 52.055,
15
+ "eval_samples_per_second": 1.594,
16
+ "eval_steps_per_second": 0.807,
17
+ "step": 1
18
+ },
19
+ {
20
+ "epoch": 1.88,
21
+ "eval_loss": 2.2997043132781982,
22
+ "eval_runtime": 52.1609,
23
+ "eval_samples_per_second": 1.591,
24
+ "eval_steps_per_second": 0.805,
25
+ "step": 3
26
+ },
27
+ {
28
+ "epoch": 2.51,
29
+ "eval_loss": 2.2976605892181396,
30
+ "eval_runtime": 52.0558,
31
+ "eval_samples_per_second": 1.594,
32
+ "eval_steps_per_second": 0.807,
33
+ "step": 4
34
+ },
35
+ {
36
+ "epoch": 3.14,
37
+ "grad_norm": 0.0927734375,
38
+ "learning_rate": 1.9238795325112867e-05,
39
+ "loss": 2.2951,
40
+ "step": 5
41
+ },
42
+ {
43
+ "epoch": 3.76,
44
+ "eval_loss": 2.293534278869629,
45
+ "eval_runtime": 52.2245,
46
+ "eval_samples_per_second": 1.589,
47
+ "eval_steps_per_second": 0.804,
48
+ "step": 6
49
+ },
50
+ {
51
+ "epoch": 4.39,
52
+ "eval_loss": 2.290515184402466,
53
+ "eval_runtime": 52.068,
54
+ "eval_samples_per_second": 1.594,
55
+ "eval_steps_per_second": 0.807,
56
+ "step": 7
57
+ },
58
+ {
59
+ "epoch": 5.65,
60
+ "eval_loss": 2.2852978706359863,
61
+ "eval_runtime": 52.0381,
62
+ "eval_samples_per_second": 1.595,
63
+ "eval_steps_per_second": 0.807,
64
+ "step": 9
65
+ },
66
+ {
67
+ "epoch": 6.27,
68
+ "grad_norm": 0.095703125,
69
+ "learning_rate": 1.7071067811865477e-05,
70
+ "loss": 2.2845,
71
+ "step": 10
72
+ },
73
+ {
74
+ "epoch": 6.9,
75
+ "eval_loss": 2.2788708209991455,
76
+ "eval_runtime": 52.4054,
77
+ "eval_samples_per_second": 1.584,
78
+ "eval_steps_per_second": 0.801,
79
+ "step": 11
80
+ },
81
+ {
82
+ "epoch": 7.53,
83
+ "eval_loss": 2.275712251663208,
84
+ "eval_runtime": 52.0276,
85
+ "eval_samples_per_second": 1.595,
86
+ "eval_steps_per_second": 0.807,
87
+ "step": 12
88
+ },
89
+ {
90
+ "epoch": 8.78,
91
+ "eval_loss": 2.2695822715759277,
92
+ "eval_runtime": 52.0264,
93
+ "eval_samples_per_second": 1.595,
94
+ "eval_steps_per_second": 0.807,
95
+ "step": 14
96
+ },
97
+ {
98
+ "epoch": 9.41,
99
+ "grad_norm": 0.09375,
100
+ "learning_rate": 1.3826834323650899e-05,
101
+ "loss": 2.2729,
102
+ "step": 15
103
+ },
104
+ {
105
+ "epoch": 9.41,
106
+ "eval_loss": 2.266735553741455,
107
+ "eval_runtime": 52.0513,
108
+ "eval_samples_per_second": 1.595,
109
+ "eval_steps_per_second": 0.807,
110
+ "step": 15
111
+ },
112
+ {
113
+ "epoch": 10.67,
114
+ "eval_loss": 2.2618672847747803,
115
+ "eval_runtime": 52.0274,
116
+ "eval_samples_per_second": 1.595,
117
+ "eval_steps_per_second": 0.807,
118
+ "step": 17
119
+ },
120
+ {
121
+ "epoch": 11.92,
122
+ "eval_loss": 2.2575833797454834,
123
+ "eval_runtime": 52.0229,
124
+ "eval_samples_per_second": 1.595,
125
+ "eval_steps_per_second": 0.807,
126
+ "step": 19
127
+ },
128
+ {
129
+ "epoch": 12.55,
130
+ "grad_norm": 0.08984375,
131
+ "learning_rate": 1e-05,
132
+ "loss": 2.2599,
133
+ "step": 20
134
+ },
135
+ {
136
+ "epoch": 12.55,
137
+ "eval_loss": 2.2554843425750732,
138
+ "eval_runtime": 52.0166,
139
+ "eval_samples_per_second": 1.596,
140
+ "eval_steps_per_second": 0.807,
141
+ "step": 20
142
+ },
143
+ {
144
+ "epoch": 13.8,
145
+ "eval_loss": 2.2518961429595947,
146
+ "eval_runtime": 52.0251,
147
+ "eval_samples_per_second": 1.595,
148
+ "eval_steps_per_second": 0.807,
149
+ "step": 22
150
+ },
151
+ {
152
+ "epoch": 14.43,
153
+ "eval_loss": 2.250413179397583,
154
+ "eval_runtime": 52.2375,
155
+ "eval_samples_per_second": 1.589,
156
+ "eval_steps_per_second": 0.804,
157
+ "step": 23
158
+ },
159
+ {
160
+ "epoch": 15.69,
161
+ "grad_norm": 0.0869140625,
162
+ "learning_rate": 6.173165676349103e-06,
163
+ "loss": 2.2523,
164
+ "step": 25
165
+ },
166
+ {
167
+ "epoch": 15.69,
168
+ "eval_loss": 2.2478551864624023,
169
+ "eval_runtime": 52.034,
170
+ "eval_samples_per_second": 1.595,
171
+ "eval_steps_per_second": 0.807,
172
+ "step": 25
173
+ },
174
+ {
175
+ "epoch": 16.94,
176
+ "eval_loss": 2.2462329864501953,
177
+ "eval_runtime": 52.0568,
178
+ "eval_samples_per_second": 1.594,
179
+ "eval_steps_per_second": 0.807,
180
+ "step": 27
181
+ },
182
+ {
183
+ "epoch": 17.57,
184
+ "eval_loss": 2.245408296585083,
185
+ "eval_runtime": 52.0465,
186
+ "eval_samples_per_second": 1.595,
187
+ "eval_steps_per_second": 0.807,
188
+ "step": 28
189
+ },
190
+ {
191
+ "epoch": 18.82,
192
+ "grad_norm": 0.087890625,
193
+ "learning_rate": 2.9289321881345257e-06,
194
+ "loss": 2.2471,
195
+ "step": 30
196
+ },
197
+ {
198
+ "epoch": 18.82,
199
+ "eval_loss": 2.2442374229431152,
200
+ "eval_runtime": 52.0648,
201
+ "eval_samples_per_second": 1.594,
202
+ "eval_steps_per_second": 0.807,
203
+ "step": 30
204
+ },
205
+ {
206
+ "epoch": 19.45,
207
+ "eval_loss": 2.2436985969543457,
208
+ "eval_runtime": 52.0526,
209
+ "eval_samples_per_second": 1.595,
210
+ "eval_steps_per_second": 0.807,
211
+ "step": 31
212
+ },
213
+ {
214
+ "epoch": 20.71,
215
+ "eval_loss": 2.2432022094726562,
216
+ "eval_runtime": 52.0653,
217
+ "eval_samples_per_second": 1.594,
218
+ "eval_steps_per_second": 0.807,
219
+ "step": 33
220
+ },
221
+ {
222
+ "epoch": 21.96,
223
+ "grad_norm": 0.08349609375,
224
+ "learning_rate": 7.612046748871327e-07,
225
+ "loss": 2.2444,
226
+ "step": 35
227
+ },
228
+ {
229
+ "epoch": 21.96,
230
+ "eval_loss": 2.24267315864563,
231
+ "eval_runtime": 52.0344,
232
+ "eval_samples_per_second": 1.595,
233
+ "eval_steps_per_second": 0.807,
234
+ "step": 35
235
+ },
236
+ {
237
+ "epoch": 22.59,
238
+ "eval_loss": 2.2428243160247803,
239
+ "eval_runtime": 52.041,
240
+ "eval_samples_per_second": 1.595,
241
+ "eval_steps_per_second": 0.807,
242
+ "step": 36
243
+ },
244
+ {
245
+ "epoch": 23.84,
246
+ "eval_loss": 2.2428712844848633,
247
+ "eval_runtime": 52.0448,
248
+ "eval_samples_per_second": 1.595,
249
+ "eval_steps_per_second": 0.807,
250
+ "step": 38
251
+ },
252
+ {
253
+ "epoch": 24.47,
254
+ "eval_loss": 2.2428550720214844,
255
+ "eval_runtime": 52.0283,
256
+ "eval_samples_per_second": 1.595,
257
+ "eval_steps_per_second": 0.807,
258
+ "step": 39
259
+ },
260
+ {
261
+ "epoch": 25.1,
262
+ "grad_norm": 0.08642578125,
263
+ "learning_rate": 0.0,
264
+ "loss": 2.2427,
265
+ "step": 40
266
+ },
267
+ {
268
+ "epoch": 25.1,
269
+ "eval_loss": 2.2429935932159424,
270
+ "eval_runtime": 52.018,
271
+ "eval_samples_per_second": 1.596,
272
+ "eval_steps_per_second": 0.807,
273
+ "step": 40
274
+ },
275
+ {
276
+ "epoch": 25.1,
277
+ "step": 40,
278
+ "total_flos": 5.866939585627423e+18,
279
+ "train_loss": 2.2623767852783203,
280
+ "train_runtime": 20272.0714,
281
+ "train_samples_per_second": 0.805,
282
+ "train_steps_per_second": 0.002
283
+ }
284
+ ],
285
+ "logging_steps": 5,
286
+ "max_steps": 40,
287
+ "num_input_tokens_seen": 0,
288
+ "num_train_epochs": 40,
289
+ "save_steps": 500,
290
+ "total_flos": 5.866939585627423e+18,
291
+ "train_batch_size": 2,
292
+ "trial_name": null,
293
+ "trial_params": null
294
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c71e8cca824dc7a17386b88759195bb54283717f34d4ccd2d319e299034ef15d
3
+ size 4984