terry69 commited on
Commit
9610215
1 Parent(s): fbd6d64

Model save

Browse files
Files changed (28) hide show
  1. README.md +67 -0
  2. adapter_config.json +34 -0
  3. adapter_model.safetensors +3 -0
  4. all_results.json +9 -0
  5. runs/May27_21-34-18_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716845862.ip-172-31-69-60.ec2.internal.23523.0 +3 -0
  6. runs/May27_21-50-49_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846661.ip-172-31-69-60.ec2.internal.12422.0 +3 -0
  7. runs/May27_21-51-46_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846718.ip-172-31-69-60.ec2.internal.14769.0 +3 -0
  8. runs/May27_21-52-49_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846780.ip-172-31-69-60.ec2.internal.17185.0 +3 -0
  9. runs/May27_21-53-25_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846816.ip-172-31-69-60.ec2.internal.18768.0 +3 -0
  10. runs/May27_21-54-02_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846854.ip-172-31-69-60.ec2.internal.20423.0 +3 -0
  11. runs/May27_21-54-40_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846892.ip-172-31-69-60.ec2.internal.22058.0 +3 -0
  12. runs/May27_21-57-52_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716847084.ip-172-31-69-60.ec2.internal.28976.0 +3 -0
  13. runs/May27_21-58-59_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716847151.ip-172-31-69-60.ec2.internal.31493.0 +3 -0
  14. runs/May27_22-08-59_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716847751.ip-172-31-69-60.ec2.internal.4939.0 +3 -0
  15. runs/May27_22-11-24_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716847896.ip-172-31-69-60.ec2.internal.9970.0 +3 -0
  16. runs/May27_22-14-02_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848053.ip-172-31-69-60.ec2.internal.14045.0 +3 -0
  17. runs/May27_22-15-16_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848128.ip-172-31-69-60.ec2.internal.16064.0 +3 -0
  18. runs/May27_22-17-27_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848272.ip-172-31-69-60.ec2.internal.19273.0 +3 -0
  19. runs/May27_22-19-18_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848375.ip-172-31-69-60.ec2.internal.21785.0 +3 -0
  20. runs/May27_22-20-22_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848439.ip-172-31-69-60.ec2.internal.23944.0 +3 -0
  21. runs/May27_22-22-05_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848542.ip-172-31-69-60.ec2.internal.26567.0 +3 -0
  22. special_tokens_map.json +24 -0
  23. tokenizer.json +0 -0
  24. tokenizer.model +3 -0
  25. tokenizer_config.json +43 -0
  26. train_results.json +9 -0
  27. trainer_state.json +393 -0
  28. training_args.bin +3 -0
README.md ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ library_name: peft
4
+ tags:
5
+ - trl
6
+ - sft
7
+ - generated_from_trainer
8
+ base_model: mistralai/Mistral-7B-v0.1
9
+ model-index:
10
+ - name: mistral5p
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # mistral5p
18
+
19
+ This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
20
+ It achieves the following results on the evaluation set:
21
+ - Loss: nan
22
+
23
+ ## Model description
24
+
25
+ More information needed
26
+
27
+ ## Intended uses & limitations
28
+
29
+ More information needed
30
+
31
+ ## Training and evaluation data
32
+
33
+ More information needed
34
+
35
+ ## Training procedure
36
+
37
+ ### Training hyperparameters
38
+
39
+ The following hyperparameters were used during training:
40
+ - learning_rate: 0.0002
41
+ - train_batch_size: 8
42
+ - eval_batch_size: 1
43
+ - seed: 42
44
+ - distributed_type: multi-GPU
45
+ - num_devices: 4
46
+ - gradient_accumulation_steps: 4
47
+ - total_train_batch_size: 128
48
+ - total_eval_batch_size: 4
49
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
50
+ - lr_scheduler_type: cosine
51
+ - lr_scheduler_warmup_ratio: 0.1
52
+ - num_epochs: 1
53
+
54
+ ### Training results
55
+
56
+ | Training Loss | Epoch | Step | Validation Loss |
57
+ |:-------------:|:------:|:----:|:---------------:|
58
+ | 0.7092 | 0.9969 | 243 | nan |
59
+
60
+
61
+ ### Framework versions
62
+
63
+ - PEFT 0.11.1
64
+ - Transformers 4.41.1
65
+ - Pytorch 2.2.2+cu121
66
+ - Datasets 2.19.1
67
+ - Tokenizers 0.19.1
adapter_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "mistralai/Mistral-7B-v0.1",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 8,
14
+ "lora_dropout": 0.05,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 6,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "k_proj",
24
+ "gate_proj",
25
+ "o_proj",
26
+ "q_proj",
27
+ "down_proj",
28
+ "up_proj",
29
+ "v_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "use_dora": false,
33
+ "use_rslora": false
34
+ }
adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fe90f2f98208e74edd203b571c5549da46239bd0283d4b96ce66fefe4c7662e
3
+ size 31516744
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.9969230769230769,
3
+ "total_flos": 5623477606285312.0,
4
+ "train_loss": 0.7391852758548878,
5
+ "train_runtime": 18302.1626,
6
+ "train_samples": 31180,
7
+ "train_samples_per_second": 1.704,
8
+ "train_steps_per_second": 0.013
9
+ }
runs/May27_21-34-18_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716845862.ip-172-31-69-60.ec2.internal.23523.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9aada4a7ded10c996b91d89d1038c9d6afa059db9a60396875228bd2cadc4bf
3
+ size 5572
runs/May27_21-50-49_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846661.ip-172-31-69-60.ec2.internal.12422.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:685a46cf146655868a9c7c816a1407092b7ae9e7c2d2463e7fe76988dc6d1815
3
+ size 5986
runs/May27_21-51-46_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846718.ip-172-31-69-60.ec2.internal.14769.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5c235543a273abfab4a0dd02f710aaa75cd4bffa34fe4427025257619af23b3
3
+ size 5572
runs/May27_21-52-49_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846780.ip-172-31-69-60.ec2.internal.17185.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5678cbb9bc0e044dd777bd532c4e6b0a35d047e1189643add443beb71f2bbdd8
3
+ size 5572
runs/May27_21-53-25_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846816.ip-172-31-69-60.ec2.internal.18768.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a56fdc42bc86ce1f9f4d2292c12ef556b5c654b28455e31522da1ea1a7ceed2
3
+ size 5572
runs/May27_21-54-02_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846854.ip-172-31-69-60.ec2.internal.20423.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9c0ecd45778978ae693e63ae10a1380c32936acd8df595cdfefdd5895a5ceb4
3
+ size 5779
runs/May27_21-54-40_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716846892.ip-172-31-69-60.ec2.internal.22058.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:efbc29668edf80f66fd2c9fdfad8efa18909e53179d79558f9094213f7733a78
3
+ size 5779
runs/May27_21-57-52_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716847084.ip-172-31-69-60.ec2.internal.28976.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed62cd56c05e39c6220bfed90bb4586299ec6b228ca01858819f1ef85fd7d2c0
3
+ size 5986
runs/May27_21-58-59_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716847151.ip-172-31-69-60.ec2.internal.31493.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7de6497d021dba6f9e1ddca9db4b3ba92c61d71071f538c0051d7d1b50fcf6d7
3
+ size 5572
runs/May27_22-08-59_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716847751.ip-172-31-69-60.ec2.internal.4939.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2a46949ec2aa63756c8035a9ca6d575538cfd03ce8aa14c353de0bca24e331a
3
+ size 5572
runs/May27_22-11-24_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716847896.ip-172-31-69-60.ec2.internal.9970.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd6e6b08da14c3893e6e1e49c80610339befd2d5b85901abfa7171171d4416ed
3
+ size 5572
runs/May27_22-14-02_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848053.ip-172-31-69-60.ec2.internal.14045.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:34842942951fc88c3f4b4bb08a82d2ef804f410ddb067136dc4aad95b65576e1
3
+ size 5986
runs/May27_22-15-16_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848128.ip-172-31-69-60.ec2.internal.16064.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbadc2f17fe22a11779d81ea25a4a4e71b7eba34a3b7c215c76db59308d23e7c
3
+ size 5572
runs/May27_22-17-27_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848272.ip-172-31-69-60.ec2.internal.19273.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:685d65721234f727b412b2430e4b97639e452c958e8eb1b626ca60da6fc9ac51
3
+ size 5986
runs/May27_22-19-18_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848375.ip-172-31-69-60.ec2.internal.21785.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c4a080c09bd356ea0915e82e47e65a430773626cde4730374120b415ed7f27b
3
+ size 5297
runs/May27_22-20-22_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848439.ip-172-31-69-60.ec2.internal.23944.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ca769519f176451f3c8064f5d5c1e25154c69c9563dd89806e9cde06356b871
3
+ size 5297
runs/May27_22-22-05_ip-172-31-69-60.ec2.internal/events.out.tfevents.1716848542.ip-172-31-69-60.ec2.internal.26567.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aabc977d6457a22fd50b73f0d67dfef24963b94f6c128b9d9f0ccd5e437ce377
3
+ size 15950
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
+ size 493443
tokenizer_config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ }
29
+ },
30
+ "additional_special_tokens": [],
31
+ "bos_token": "<s>",
32
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
33
+ "clean_up_tokenization_spaces": false,
34
+ "eos_token": "</s>",
35
+ "legacy": true,
36
+ "model_max_length": 2048,
37
+ "pad_token": "</s>",
38
+ "sp_model_kwargs": {},
39
+ "spaces_between_special_tokens": false,
40
+ "tokenizer_class": "LlamaTokenizer",
41
+ "unk_token": "<unk>",
42
+ "use_default_system_prompt": false
43
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 0.9969230769230769,
3
+ "total_flos": 5623477606285312.0,
4
+ "train_loss": 0.7391852758548878,
5
+ "train_runtime": 18302.1626,
6
+ "train_samples": 31180,
7
+ "train_samples_per_second": 1.704,
8
+ "train_steps_per_second": 0.013
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,393 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.9969230769230769,
5
+ "eval_steps": 500,
6
+ "global_step": 243,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0041025641025641026,
13
+ "grad_norm": 0.26542961092188894,
14
+ "learning_rate": 8.000000000000001e-06,
15
+ "loss": 0.8534,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.020512820512820513,
20
+ "grad_norm": 0.2656899378598073,
21
+ "learning_rate": 4e-05,
22
+ "loss": 0.8667,
23
+ "step": 5
24
+ },
25
+ {
26
+ "epoch": 0.041025641025641026,
27
+ "grad_norm": 0.2572181527992512,
28
+ "learning_rate": 8e-05,
29
+ "loss": 0.8031,
30
+ "step": 10
31
+ },
32
+ {
33
+ "epoch": 0.06153846153846154,
34
+ "grad_norm": 0.18710816410634198,
35
+ "learning_rate": 0.00012,
36
+ "loss": 0.7552,
37
+ "step": 15
38
+ },
39
+ {
40
+ "epoch": 0.08205128205128205,
41
+ "grad_norm": 0.17366628242950644,
42
+ "learning_rate": 0.00016,
43
+ "loss": 0.7434,
44
+ "step": 20
45
+ },
46
+ {
47
+ "epoch": 0.10256410256410256,
48
+ "grad_norm": 0.18209587440813843,
49
+ "learning_rate": 0.0002,
50
+ "loss": 0.7613,
51
+ "step": 25
52
+ },
53
+ {
54
+ "epoch": 0.12307692307692308,
55
+ "grad_norm": 0.17584855523843826,
56
+ "learning_rate": 0.00019974051702905277,
57
+ "loss": 0.7686,
58
+ "step": 30
59
+ },
60
+ {
61
+ "epoch": 0.14358974358974358,
62
+ "grad_norm": 0.1876526868957483,
63
+ "learning_rate": 0.00019896341474445525,
64
+ "loss": 0.7408,
65
+ "step": 35
66
+ },
67
+ {
68
+ "epoch": 0.1641025641025641,
69
+ "grad_norm": 0.14962305137903897,
70
+ "learning_rate": 0.00019767272604239824,
71
+ "loss": 0.7422,
72
+ "step": 40
73
+ },
74
+ {
75
+ "epoch": 0.18461538461538463,
76
+ "grad_norm": 0.16336671883236106,
77
+ "learning_rate": 0.00019587514915766124,
78
+ "loss": 0.7565,
79
+ "step": 45
80
+ },
81
+ {
82
+ "epoch": 0.20512820512820512,
83
+ "grad_norm": 0.15152018240587575,
84
+ "learning_rate": 0.00019358001290205543,
85
+ "loss": 0.7493,
86
+ "step": 50
87
+ },
88
+ {
89
+ "epoch": 0.22564102564102564,
90
+ "grad_norm": 0.14810534949892065,
91
+ "learning_rate": 0.0001907992282510675,
92
+ "loss": 0.7539,
93
+ "step": 55
94
+ },
95
+ {
96
+ "epoch": 0.24615384615384617,
97
+ "grad_norm": 0.16847734140964582,
98
+ "learning_rate": 0.00018754722652995347,
99
+ "loss": 0.7395,
100
+ "step": 60
101
+ },
102
+ {
103
+ "epoch": 0.26666666666666666,
104
+ "grad_norm": 0.15436628692835286,
105
+ "learning_rate": 0.00018384088452007578,
106
+ "loss": 0.747,
107
+ "step": 65
108
+ },
109
+ {
110
+ "epoch": 0.28717948717948716,
111
+ "grad_norm": 0.15652291403007046,
112
+ "learning_rate": 0.00017969943687415576,
113
+ "loss": 0.7506,
114
+ "step": 70
115
+ },
116
+ {
117
+ "epoch": 0.3076923076923077,
118
+ "grad_norm": 0.16413347022113,
119
+ "learning_rate": 0.0001751443762949772,
120
+ "loss": 0.7611,
121
+ "step": 75
122
+ },
123
+ {
124
+ "epoch": 0.3282051282051282,
125
+ "grad_norm": 0.16182771329972745,
126
+ "learning_rate": 0.00017019934199557867,
127
+ "loss": 0.7576,
128
+ "step": 80
129
+ },
130
+ {
131
+ "epoch": 0.3487179487179487,
132
+ "grad_norm": 0.137569364566755,
133
+ "learning_rate": 0.00016488999701978903,
134
+ "loss": 0.7451,
135
+ "step": 85
136
+ },
137
+ {
138
+ "epoch": 0.36923076923076925,
139
+ "grad_norm": 0.14900540538806303,
140
+ "learning_rate": 0.00015924389505977038,
141
+ "loss": 0.7197,
142
+ "step": 90
143
+ },
144
+ {
145
+ "epoch": 0.38974358974358975,
146
+ "grad_norm": 0.14655990407738764,
147
+ "learning_rate": 0.00015329033746173975,
148
+ "loss": 0.7149,
149
+ "step": 95
150
+ },
151
+ {
152
+ "epoch": 0.41025641025641024,
153
+ "grad_norm": 0.1402768270199422,
154
+ "learning_rate": 0.00014706022116196208,
155
+ "loss": 0.7018,
156
+ "step": 100
157
+ },
158
+ {
159
+ "epoch": 0.4307692307692308,
160
+ "grad_norm": 0.14134928021057855,
161
+ "learning_rate": 0.00014058587834217355,
162
+ "loss": 0.7324,
163
+ "step": 105
164
+ },
165
+ {
166
+ "epoch": 0.4512820512820513,
167
+ "grad_norm": 0.15600188868890955,
168
+ "learning_rate": 0.00013390090863657047,
169
+ "loss": 0.748,
170
+ "step": 110
171
+ },
172
+ {
173
+ "epoch": 0.4717948717948718,
174
+ "grad_norm": 0.13996117144988574,
175
+ "learning_rate": 0.0001270400047611508,
176
+ "loss": 0.7703,
177
+ "step": 115
178
+ },
179
+ {
180
+ "epoch": 0.49230769230769234,
181
+ "grad_norm": 0.1510982087519504,
182
+ "learning_rate": 0.00012003877247033411,
183
+ "loss": 0.7515,
184
+ "step": 120
185
+ },
186
+ {
187
+ "epoch": 0.5128205128205128,
188
+ "grad_norm": 0.14250885612041378,
189
+ "learning_rate": 0.00011293354577522263,
190
+ "loss": 0.7196,
191
+ "step": 125
192
+ },
193
+ {
194
+ "epoch": 0.5333333333333333,
195
+ "grad_norm": 0.14795873562666287,
196
+ "learning_rate": 0.00010576119838245844,
197
+ "loss": 0.731,
198
+ "step": 130
199
+ },
200
+ {
201
+ "epoch": 0.5538461538461539,
202
+ "grad_norm": 0.13063022182064904,
203
+ "learning_rate": 9.85589523322443e-05,
204
+ "loss": 0.7301,
205
+ "step": 135
206
+ },
207
+ {
208
+ "epoch": 0.5743589743589743,
209
+ "grad_norm": 0.13557487318693218,
210
+ "learning_rate": 9.136418482863229e-05,
211
+ "loss": 0.718,
212
+ "step": 140
213
+ },
214
+ {
215
+ "epoch": 0.5948717948717949,
216
+ "grad_norm": 0.15507641481392034,
217
+ "learning_rate": 8.42142342645646e-05,
218
+ "loss": 0.7193,
219
+ "step": 145
220
+ },
221
+ {
222
+ "epoch": 0.6153846153846154,
223
+ "grad_norm": 0.15494002592225475,
224
+ "learning_rate": 7.714620644833111e-05,
225
+ "loss": 0.731,
226
+ "step": 150
227
+ },
228
+ {
229
+ "epoch": 0.6358974358974359,
230
+ "grad_norm": 0.1601203804358568,
231
+ "learning_rate": 7.019678203706163e-05,
232
+ "loss": 0.75,
233
+ "step": 155
234
+ },
235
+ {
236
+ "epoch": 0.6564102564102564,
237
+ "grad_norm": 0.15685275896075176,
238
+ "learning_rate": 6.340202617660842e-05,
239
+ "loss": 0.7505,
240
+ "step": 160
241
+ },
242
+ {
243
+ "epoch": 0.676923076923077,
244
+ "grad_norm": 0.1416704035001306,
245
+ "learning_rate": 5.679720133572206e-05,
246
+ "loss": 0.7311,
247
+ "step": 165
248
+ },
249
+ {
250
+ "epoch": 0.6974358974358974,
251
+ "grad_norm": 0.14126335283382643,
252
+ "learning_rate": 5.0416584305848524e-05,
253
+ "loss": 0.755,
254
+ "step": 170
255
+ },
256
+ {
257
+ "epoch": 0.717948717948718,
258
+ "grad_norm": 0.13481365291511518,
259
+ "learning_rate": 4.4293288316255653e-05,
260
+ "loss": 0.695,
261
+ "step": 175
262
+ },
263
+ {
264
+ "epoch": 0.7384615384615385,
265
+ "grad_norm": 0.14785228000969083,
266
+ "learning_rate": 3.845909118765073e-05,
267
+ "loss": 0.7209,
268
+ "step": 180
269
+ },
270
+ {
271
+ "epoch": 0.7589743589743589,
272
+ "grad_norm": 0.1503268472656009,
273
+ "learning_rate": 3.294427041611425e-05,
274
+ "loss": 0.7307,
275
+ "step": 185
276
+ },
277
+ {
278
+ "epoch": 0.7794871794871795,
279
+ "grad_norm": 0.15299701883959732,
280
+ "learning_rate": 2.7777446043207058e-05,
281
+ "loss": 0.7351,
282
+ "step": 190
283
+ },
284
+ {
285
+ "epoch": 0.8,
286
+ "grad_norm": 0.1373494159366909,
287
+ "learning_rate": 2.2985432127701946e-05,
288
+ "loss": 0.7304,
289
+ "step": 195
290
+ },
291
+ {
292
+ "epoch": 0.8205128205128205,
293
+ "grad_norm": 0.16245499908137687,
294
+ "learning_rate": 1.859309758975132e-05,
295
+ "loss": 0.7385,
296
+ "step": 200
297
+ },
298
+ {
299
+ "epoch": 0.841025641025641,
300
+ "grad_norm": 0.13807085431435603,
301
+ "learning_rate": 1.462323714966114e-05,
302
+ "loss": 0.714,
303
+ "step": 205
304
+ },
305
+ {
306
+ "epoch": 0.8615384615384616,
307
+ "grad_norm": 0.12926916684005163,
308
+ "learning_rate": 1.1096453031056264e-05,
309
+ "loss": 0.7078,
310
+ "step": 210
311
+ },
312
+ {
313
+ "epoch": 0.882051282051282,
314
+ "grad_norm": 0.12918592796308734,
315
+ "learning_rate": 8.031048042356392e-06,
316
+ "loss": 0.7208,
317
+ "step": 215
318
+ },
319
+ {
320
+ "epoch": 0.9025641025641026,
321
+ "grad_norm": 0.14837431396022016,
322
+ "learning_rate": 5.442930591433992e-06,
323
+ "loss": 0.7435,
324
+ "step": 220
325
+ },
326
+ {
327
+ "epoch": 0.9230769230769231,
328
+ "grad_norm": 0.14838485907528307,
329
+ "learning_rate": 3.3455321263955786e-06,
330
+ "loss": 0.7255,
331
+ "step": 225
332
+ },
333
+ {
334
+ "epoch": 0.9435897435897436,
335
+ "grad_norm": 0.1480931290003826,
336
+ "learning_rate": 1.7497374309405346e-06,
337
+ "loss": 0.695,
338
+ "step": 230
339
+ },
340
+ {
341
+ "epoch": 0.9641025641025641,
342
+ "grad_norm": 0.14733343614150893,
343
+ "learning_rate": 6.638281360408339e-07,
344
+ "loss": 0.7012,
345
+ "step": 235
346
+ },
347
+ {
348
+ "epoch": 0.9846153846153847,
349
+ "grad_norm": 0.1368626907349573,
350
+ "learning_rate": 9.343974109685682e-08,
351
+ "loss": 0.7092,
352
+ "step": 240
353
+ },
354
+ {
355
+ "epoch": 0.9969230769230769,
356
+ "eval_loss": NaN,
357
+ "eval_runtime": 748.4122,
358
+ "eval_samples_per_second": 1.545,
359
+ "eval_steps_per_second": 0.386,
360
+ "step": 243
361
+ },
362
+ {
363
+ "epoch": 0.9969230769230769,
364
+ "step": 243,
365
+ "total_flos": 5623477606285312.0,
366
+ "train_loss": 0.7391852758548878,
367
+ "train_runtime": 18302.1626,
368
+ "train_samples_per_second": 1.704,
369
+ "train_steps_per_second": 0.013
370
+ }
371
+ ],
372
+ "logging_steps": 5,
373
+ "max_steps": 243,
374
+ "num_input_tokens_seen": 0,
375
+ "num_train_epochs": 1,
376
+ "save_steps": 1000,
377
+ "stateful_callbacks": {
378
+ "TrainerControl": {
379
+ "args": {
380
+ "should_epoch_stop": false,
381
+ "should_evaluate": false,
382
+ "should_log": false,
383
+ "should_save": false,
384
+ "should_training_stop": false
385
+ },
386
+ "attributes": {}
387
+ }
388
+ },
389
+ "total_flos": 5623477606285312.0,
390
+ "train_batch_size": 8,
391
+ "trial_name": null,
392
+ "trial_params": null
393
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92484e9a5c8df8e8c9f43c40230cc189f21535aa92db5a37bb10f16e93a8e979
3
+ size 6456