Hugging Face's logo

chrisgru
/
llama-7b-v0

Text Generation
Transformers
TensorBoard
llama
Generated from Trainer
text-generation-inference
Inference Endpoints
4-bit precision
bitsandbytes
Model card Files Metrics Community
chrisgru commited on
Commit
f9390de
•
1 Parent(s): 97505f9

Upload folder using huggingface_hub

Browse files
Files changed (19) hide show
  1. README.md +77 -0
  2. adapter_config.json +31 -0
  3. adapter_model.bin +3 -0
  4. added_tokens.json +4 -0
  5. config.json +39 -0
  6. runs/Nov29_15-01-05_8085f7d2ae17/events.out.tfevents.1701270065.8085f7d2ae17.9471.0 +3 -0
  7. runs/Nov29_15-18-35_8085f7d2ae17/events.out.tfevents.1701271116.8085f7d2ae17.12404.0 +3 -0
  8. runs/Nov29_15-22-59_8085f7d2ae17/events.out.tfevents.1701271379.8085f7d2ae17.13442.0 +3 -0
  9. runs/Nov29_15-24-31_8085f7d2ae17/events.out.tfevents.1701271471.8085f7d2ae17.14044.0 +3 -0
  10. runs/Nov29_15-25-59_8085f7d2ae17/events.out.tfevents.1701271560.8085f7d2ae17.14575.0 +3 -0
  11. runs/Nov29_15-27-09_8085f7d2ae17/events.out.tfevents.1701271629.8085f7d2ae17.14877.0 +3 -0
  12. runs/Nov29_15-31-46_8085f7d2ae17/events.out.tfevents.1701271907.8085f7d2ae17.15995.0 +3 -0
  13. runs/Nov29_15-33-02_8085f7d2ae17/events.out.tfevents.1701271982.8085f7d2ae17.16810.0 +3 -0
  14. runs/Nov29_16-16-27_8085f7d2ae17/events.out.tfevents.1701274587.8085f7d2ae17.22473.0 +3 -0
  15. runs/Nov29_16-18-08_8085f7d2ae17/events.out.tfevents.1701274689.8085f7d2ae17.22878.0 +3 -0
  16. runs/Nov29_16-22-40_8085f7d2ae17/events.out.tfevents.1701274960.8085f7d2ae17.23986.0 +3 -0
  17. special_tokens_map.json +24 -0
  18. tokenizer.model +3 -0
  19. tokenizer_config.json +60 -0
README.md ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: meta-llama/Llama-2-7b-hf
3
+ tags:
4
+ - generated_from_trainer
5
+ model-index:
6
+ - name: model_output
7
+ results: []
8
+ ---
9
+
10
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
11
+ should probably proofread and complete it, then remove this comment. -->
12
+
13
+ [<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
14
+ # model_output
15
+
16
+ This model is a fine-tuned version of [meta-llama/Llama-2-7b-hf](https://huggingface.co/meta-llama/Llama-2-7b-hf) on the None dataset.
17
+ It achieves the following results on the evaluation set:
18
+ - Loss: 0.6505
19
+
20
+ ## Model description
21
+
22
+ More information needed
23
+
24
+ ## Intended uses & limitations
25
+
26
+ More information needed
27
+
28
+ ## Training and evaluation data
29
+
30
+ More information needed
31
+
32
+ ## Training procedure
33
+
34
+ ### Training hyperparameters
35
+
36
+ The following hyperparameters were used during training:
37
+ - learning_rate: 3e-05
38
+ - train_batch_size: 4
39
+ - eval_batch_size: 4
40
+ - seed: 42
41
+ - gradient_accumulation_steps: 4
42
+ - total_train_batch_size: 16
43
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
44
+ - lr_scheduler_type: constant
45
+ - lr_scheduler_warmup_steps: 100
46
+ - num_epochs: 2
47
+
48
+ ### Training results
49
+
50
+ | Training Loss | Epoch | Step | Validation Loss |
51
+ |:-------------:|:-----:|:----:|:---------------:|
52
+ | 0.9209 | 0.11 | 20 | 0.8467 |
53
+ | 0.767 | 0.22 | 40 | 0.7407 |
54
+ | 0.725 | 0.33 | 60 | 0.7076 |
55
+ | 0.689 | 0.44 | 80 | 0.6931 |
56
+ | 0.7331 | 0.55 | 100 | 0.6865 |
57
+ | 0.6744 | 0.66 | 120 | 0.6761 |
58
+ | 0.7059 | 0.77 | 140 | 0.6671 |
59
+ | 0.6711 | 0.87 | 160 | 0.6692 |
60
+ | 0.6975 | 0.98 | 180 | 0.6604 |
61
+ | 0.6039 | 1.09 | 200 | 0.6641 |
62
+ | 0.6087 | 1.2 | 220 | 0.6574 |
63
+ | 0.6164 | 1.31 | 240 | 0.6583 |
64
+ | 0.6132 | 1.42 | 260 | 0.6604 |
65
+ | 0.6136 | 1.53 | 280 | 0.6580 |
66
+ | 0.6485 | 1.64 | 300 | 0.6571 |
67
+ | 0.6205 | 1.75 | 320 | 0.6560 |
68
+ | 0.5973 | 1.86 | 340 | 0.6514 |
69
+ | 0.6048 | 1.97 | 360 | 0.6505 |
70
+
71
+
72
+ ### Framework versions
73
+
74
+ - Transformers 4.35.2
75
+ - Pytorch 2.0.1+cu118
76
+ - Datasets 2.15.0
77
+ - Tokenizers 0.15.0
adapter_config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "meta-llama/Llama-2-7b-hf",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 16,
12
+ "lora_dropout": 0.05,
13
+ "modules_to_save": [
14
+ "embed_tokens",
15
+ "lm_head"
16
+ ],
17
+ "peft_type": "LORA",
18
+ "r": 32,
19
+ "rank_pattern": {},
20
+ "revision": null,
21
+ "target_modules": [
22
+ "v_proj",
23
+ "up_proj",
24
+ "k_proj",
25
+ "down_proj",
26
+ "q_proj",
27
+ "gate_proj",
28
+ "o_proj"
29
+ ],
30
+ "task_type": "CAUSAL_LM"
31
+ }
adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73a9af85b0f14865af0019640b3e6b2f857cbbb2dd14efb9d669979e7f9d0f93
3
+ size 844298709
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "<|im_end|>": 32000,
3
+ "<|im_start|>": 32001
4
+ }
config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "meta-llama/Llama-2-7b-hf",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 32000,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 4096,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 11008,
13
+ "max_position_embeddings": 4096,
14
+ "model_type": "llama",
15
+ "num_attention_heads": 32,
16
+ "num_hidden_layers": 32,
17
+ "num_key_value_heads": 32,
18
+ "pretraining_tp": 1,
19
+ "quantization_config": {
20
+ "bnb_4bit_compute_dtype": "bfloat16",
21
+ "bnb_4bit_quant_type": "nf4",
22
+ "bnb_4bit_use_double_quant": true,
23
+ "llm_int8_enable_fp32_cpu_offload": false,
24
+ "llm_int8_has_fp16_weight": false,
25
+ "llm_int8_skip_modules": null,
26
+ "llm_int8_threshold": 6.0,
27
+ "load_in_4bit": true,
28
+ "load_in_8bit": false,
29
+ "quant_method": "bitsandbytes"
30
+ },
31
+ "rms_norm_eps": 1e-05,
32
+ "rope_scaling": null,
33
+ "rope_theta": 10000.0,
34
+ "tie_word_embeddings": false,
35
+ "torch_dtype": "float16",
36
+ "transformers_version": "4.35.2",
37
+ "use_cache": false,
38
+ "vocab_size": 32002
39
+ }
runs/Nov29_15-01-05_8085f7d2ae17/events.out.tfevents.1701270065.8085f7d2ae17.9471.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2a25a21885b8016d9ac3ffe9bef8627ff64c24364cde77cfac80d36b8d929d8
3
+ size 17600
runs/Nov29_15-18-35_8085f7d2ae17/events.out.tfevents.1701271116.8085f7d2ae17.12404.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b095ad3661077b7bf4beacfe17ac5153bb4af701686dee0eb18cb9cda2eed544
3
+ size 6094
runs/Nov29_15-22-59_8085f7d2ae17/events.out.tfevents.1701271379.8085f7d2ae17.13442.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b144587da007eb0e1287e0e429ce1e73d33121cb10d989c9129e440c78803e00
3
+ size 5058
runs/Nov29_15-24-31_8085f7d2ae17/events.out.tfevents.1701271471.8085f7d2ae17.14044.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3588ccf1e2fc123db78766d54366d25ca8b91afe35e5ffc75958994427c39eb
3
+ size 5212
runs/Nov29_15-25-59_8085f7d2ae17/events.out.tfevents.1701271560.8085f7d2ae17.14575.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d77902d57dd73a944cac9f63941377efdcf7c81681d9c1bb73f98f022a51e55
3
+ size 5212
runs/Nov29_15-27-09_8085f7d2ae17/events.out.tfevents.1701271629.8085f7d2ae17.14877.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3ff9beca7de7c110de8799cd78129934b11fcad1bee727715dde54a5fdba8b5e
3
+ size 5366
runs/Nov29_15-31-46_8085f7d2ae17/events.out.tfevents.1701271907.8085f7d2ae17.15995.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46d5678527311e16146e7c95edc9fa53ef07bff87aef18748769622d2ce6d43b
3
+ size 5058
runs/Nov29_15-33-02_8085f7d2ae17/events.out.tfevents.1701271982.8085f7d2ae17.16810.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c21bc63ed9fb645d2b54589a4e661f15c326d41f3e5b8751829c93164dbff5b
3
+ size 16095
runs/Nov29_16-16-27_8085f7d2ae17/events.out.tfevents.1701274587.8085f7d2ae17.22473.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27d07daba55ab88cf49df524ef0983de136c2c7f2c99bc16e22203368fe461bd
3
+ size 5212
runs/Nov29_16-18-08_8085f7d2ae17/events.out.tfevents.1701274689.8085f7d2ae17.22878.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6afb36439c5029a068ad8fdd201176b2887147b31898120bdd9798a60700b30
3
+ size 5058
runs/Nov29_16-22-40_8085f7d2ae17/events.out.tfevents.1701274960.8085f7d2ae17.23986.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69b2c1504827e58e839d818642a770fb4562d4dfd731eb47e3ec46d37a3bcc33
3
+ size 21646
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|im_end|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "32000": {
30
+ "content": "<|im_end|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": false
36
+ },
37
+ "32001": {
38
+ "content": "<|im_start|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": false
44
+ }
45
+ },
46
+ "bos_token": "<s>",
47
+ "clean_up_tokenization_spaces": false,
48
+ "eos_token": "<|im_end|>",
49
+ "legacy": false,
50
+ "model_max_length": 1000000000000000019884624838656,
51
+ "pad_token": "</s>",
52
+ "padding_side": "right",
53
+ "sp_model_kwargs": {},
54
+ "spaces_between_special_tokens": false,
55
+ "tokenizer_class": "LlamaTokenizer",
56
+ "trust_remote_code": true,
57
+ "unk_token": "<unk>",
58
+ "use_default_system_prompt": false,
59
+ "use_fast": true
60
+ }