Hugging Face's logo

Cem13
/
mixtralep1

PEFT
TensorBoard
Safetensors
trl
sft
Generated from Trainer
Model card Files Metrics Community
Cem13 commited on
Commit
33b0f67
•
1 Parent(s): 13ff63a

Promptengineering/mixtral2-generation

Browse files
Files changed (24) hide show
  1. README.md +61 -0
  2. adapter_config.json +31 -0
  3. adapter_model.safetensors +3 -0
  4. added_tokens.json +3 -0
  5. runs/Apr20_18-35-11_8124952fcca7/events.out.tfevents.1713638128.8124952fcca7.985.0 +3 -0
  6. runs/Apr20_18-35-11_8124952fcca7/events.out.tfevents.1713639124.8124952fcca7.985.1 +3 -0
  7. runs/Apr20_18-53-31_8124952fcca7/events.out.tfevents.1713639218.8124952fcca7.985.2 +3 -0
  8. runs/Apr20_18-56-02_8124952fcca7/events.out.tfevents.1713639371.8124952fcca7.985.3 +3 -0
  9. runs/Apr20_18-59-02_8124952fcca7/events.out.tfevents.1713639547.8124952fcca7.985.4 +3 -0
  10. runs/Apr20_19-49-33_8124952fcca7/events.out.tfevents.1713642578.8124952fcca7.985.5 +3 -0
  11. runs/Apr20_19-49-48_8124952fcca7/events.out.tfevents.1713642594.8124952fcca7.985.6 +3 -0
  12. runs/Apr20_19-57-58_8124952fcca7/events.out.tfevents.1713643085.8124952fcca7.985.7 +3 -0
  13. runs/Apr20_20-05-27_8124952fcca7/events.out.tfevents.1713643535.8124952fcca7.985.8 +3 -0
  14. runs/Apr20_20-32-46_8124952fcca7/events.out.tfevents.1713645169.8124952fcca7.985.9 +3 -0
  15. runs/Apr20_20-33-01_8124952fcca7/events.out.tfevents.1713645183.8124952fcca7.985.10 +3 -0
  16. runs/Apr20_20-33-11_8124952fcca7/events.out.tfevents.1713645196.8124952fcca7.985.11 +3 -0
  17. runs/Apr20_20-33-27_8124952fcca7/events.out.tfevents.1713645209.8124952fcca7.985.12 +3 -0
  18. runs/Apr20_20-33-37_8124952fcca7/events.out.tfevents.1713645222.8124952fcca7.985.13 +3 -0
  19. runs/Apr20_21-09-54_8124952fcca7/events.out.tfevents.1713647397.8124952fcca7.985.14 +3 -0
  20. special_tokens_map.json +30 -0
  21. tokenizer.json +0 -0
  22. tokenizer.model +3 -0
  23. tokenizer_config.json +50 -0
  24. training_args.bin +3 -0
README.md ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ library_name: peft
4
+ tags:
5
+ - trl
6
+ - sft
7
+ - generated_from_trainer
8
+ base_model: mistralai/Mixtral-8x7B-v0.1
9
+ model-index:
10
+ - name: mistral_instruct_generation
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # mistral_instruct_generation
18
+
19
+ This model is a fine-tuned version of [mistralai/Mixtral-8x7B-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-v0.1) on an unknown dataset.
20
+ It achieves the following results on the evaluation set:
21
+ - Loss: 1.0965
22
+
23
+ ## Model description
24
+
25
+ More information needed
26
+
27
+ ## Intended uses & limitations
28
+
29
+ More information needed
30
+
31
+ ## Training and evaluation data
32
+
33
+ More information needed
34
+
35
+ ## Training procedure
36
+
37
+ ### Training hyperparameters
38
+
39
+ The following hyperparameters were used during training:
40
+ - learning_rate: 0.0002
41
+ - train_batch_size: 2
42
+ - eval_batch_size: 8
43
+ - seed: 42
44
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
45
+ - lr_scheduler_type: constant
46
+ - lr_scheduler_warmup_steps: 0.03
47
+ - num_epochs: 1
48
+
49
+ ### Training results
50
+
51
+ | Training Loss | Epoch | Step | Validation Loss |
52
+ |:-------------:|:-----:|:----:|:---------------:|
53
+ | 1.0556 | 1.0 | 1278 | 1.0965 |
54
+
55
+
56
+ ### Framework versions
57
+
58
+ - PEFT 0.10.0
59
+ - Transformers 4.40.0
60
+ - Pytorch 2.2.1+cu121
61
+ - Tokenizers 0.19.1
adapter_config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "mistralai/Mixtral-8x7B-v0.1",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 16,
14
+ "lora_dropout": 0.1,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 64,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "q_proj",
24
+ "v_proj",
25
+ "o_proj",
26
+ "k_proj"
27
+ ],
28
+ "task_type": "CAUSAL_LM",
29
+ "use_dora": false,
30
+ "use_rslora": false
31
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e44ce263e6fd885f50d82ca515b9325375b43ee36ededb75acf161ce88bc2e41
3
+ size 48
added_tokens.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "[PAD]": 32000
3
+ }
runs/Apr20_18-35-11_8124952fcca7/events.out.tfevents.1713638128.8124952fcca7.985.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36dc6efc1fea5268c623e2106ce41613869a34b0b5cc816c2868b2ca8d9a1b3c
3
+ size 18709
runs/Apr20_18-35-11_8124952fcca7/events.out.tfevents.1713639124.8124952fcca7.985.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8ad2d137aa224ece31a7e11598897789e39eb30ada738b1af882339b3af1343d
3
+ size 5753
runs/Apr20_18-53-31_8124952fcca7/events.out.tfevents.1713639218.8124952fcca7.985.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0eb11bca21e8579051a4877553d22670c50950ff5c56a1e764003f7b8db120f8
3
+ size 5753
runs/Apr20_18-56-02_8124952fcca7/events.out.tfevents.1713639371.8124952fcca7.985.3 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:186b5099555dd739191d8480a6f76a250f12ca24b4629ff6e5cb3557a694e5b6
3
+ size 10998
runs/Apr20_18-59-02_8124952fcca7/events.out.tfevents.1713639547.8124952fcca7.985.4 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2096b13eeb60080382ca8bd3b2bbd10b5ddb133a6b4e247c630c20f30ddccd55
3
+ size 14752
runs/Apr20_19-49-33_8124952fcca7/events.out.tfevents.1713642578.8124952fcca7.985.5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e376fec13b362d4743b3498b97d3486be7b7eddd25e960e803e2ba46929d6b5b
3
+ size 5340
runs/Apr20_19-49-48_8124952fcca7/events.out.tfevents.1713642594.8124952fcca7.985.6 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3391ffe80479e4bef0a2ef7b9d51376dce70dd99edb518f497622fe764e79884
3
+ size 17083
runs/Apr20_19-57-58_8124952fcca7/events.out.tfevents.1713643085.8124952fcca7.985.7 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c3becf2d23ff65d6874e5b500241d6d4ffe8b1fb45c310ce73809e99c608566
3
+ size 5754
runs/Apr20_20-05-27_8124952fcca7/events.out.tfevents.1713643535.8124952fcca7.985.8 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7629ce4c535a406fc3d8416b289beb30c7f9cc3999e15cc33e62670c726ccf51
3
+ size 18159
runs/Apr20_20-32-46_8124952fcca7/events.out.tfevents.1713645169.8124952fcca7.985.9 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8ebed75065b489ca9867285676232c395a4f78644ae025e576761293d2992c5
3
+ size 5337
runs/Apr20_20-33-01_8124952fcca7/events.out.tfevents.1713645183.8124952fcca7.985.10 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1ec27e49d55542bdb22da4e4a9bd3dddbc4722f7bb9a0504d6d4b7de7ce335a
3
+ size 5337
runs/Apr20_20-33-11_8124952fcca7/events.out.tfevents.1713645196.8124952fcca7.985.11 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4bedfeaf071cdcf4e690b7cc9e99869bc2459891743fa3b8d0e6d197c71ac16c
3
+ size 5336
runs/Apr20_20-33-27_8124952fcca7/events.out.tfevents.1713645209.8124952fcca7.985.12 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:228392736ca8e44a6847b9a880a6ff6af6d587983d5d157c442dbdb62c607c89
3
+ size 5336
runs/Apr20_20-33-37_8124952fcca7/events.out.tfevents.1713645222.8124952fcca7.985.13 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c08874764f5141615e30f7321aea179fc29aa47447fc30ea9bc8f8f063a84783
3
+ size 17526
runs/Apr20_21-09-54_8124952fcca7/events.out.tfevents.1713647397.8124952fcca7.985.14 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f8f15948020191c6108144b8ea5ea218ea0c6f35c8ead0823a43846244f3d1fb
3
+ size 32710
special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "[PAD]",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
+ size 493443
tokenizer_config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "32000": {
30
+ "content": "[PAD]",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ }
37
+ },
38
+ "additional_special_tokens": [],
39
+ "bos_token": "<s>",
40
+ "clean_up_tokenization_spaces": false,
41
+ "eos_token": "</s>",
42
+ "legacy": true,
43
+ "model_max_length": 1000000000000000019884624838656,
44
+ "pad_token": "[PAD]",
45
+ "sp_model_kwargs": {},
46
+ "spaces_between_special_tokens": false,
47
+ "tokenizer_class": "LlamaTokenizer",
48
+ "unk_token": "<unk>",
49
+ "use_default_system_prompt": false
50
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5f5332d3f4e464238f0bbc2dd83e1df7a40ea760e0c73c9db2687c623ffcadd
3
+ size 4984