mtasic85 commited on
Commit
adf4b14
1 Parent(s): f4cf4d2

initial version

Browse files
config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "tangledgroup/tangled-llama-33m-32k-base-v0.1",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "bos_token_id": 1,
7
+ "eos_token_id": 2,
8
+ "hidden_size": 1024,
9
+ "intermediate_size": 4096,
10
+ "max_position_embeddings": 38400,
11
+ "model_type": "llama",
12
+ "num_attention_heads": 32,
13
+ "num_hidden_layers": 5,
14
+ "num_key_value_heads": 8,
15
+ "rms_norm_eps": 1e-05,
16
+ "rope_scaling": null,
17
+ "tie_word_embeddings": false,
18
+ "torch_dtype": "bfloat16",
19
+ "transformers_version": "4.44.2",
20
+ "use_cache": true,
21
+ "vocab_size": 38400
22
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
misc/logo.png ADDED

Git LFS Details

  • SHA256: 6db9ef3985f85ad849bd3a1051e9dadb7703dfa1fefbe50f53e0d935e10ab27b
  • Pointer size: 132 Bytes
  • Size of remote file: 1.8 MB
scripts/TRAIN.md ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Train
2
+
3
+ ## Environment
4
+
5
+ ```bash
6
+ cd scripts
7
+ python -m venv venv
8
+ source venv/bin/activate
9
+ pip install -U -r requirements.in
10
+ ```
11
+
12
+ ## Tokenizer
13
+
14
+ ```bash
15
+ python -B train_tokenizer.py
16
+ ```
17
+
18
+ ## Dataset
19
+
20
+ ```bash
21
+ python -B prepare_pretrain_dataset.py
22
+ ```
23
+
24
+ ## Model
25
+
26
+ ### Pretrain
27
+
28
+ ```bash
29
+ litgpt pretrain --config ./pretrain-model.yaml
30
+ ```
31
+
32
+ ```bash
33
+ litgpt convert_from_litgpt out/pretrain/final/ out/converted_model
34
+ cp config.json out/pretrain/final/
35
+ cp config.json out/converted_model/
36
+ ```
37
+
38
+ ```python
39
+ import torch
40
+ from safetensors.torch import save_file
41
+
42
+ state_dict = torch.load('out/converted_model/model.pth', map_location='cpu')
43
+ save_file(state_dict, 'out/converted_model/model.safetensors')
44
+ ```
45
+
46
+ ## Evaluate
47
+
48
+ ```bash
49
+ litgpt evaluate --tasks 'hellaswag,gsm8k,truthfulqa_mc2,mmlu,winogrande,arc_challenge' --out_dir 'evaluate-quick/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
50
+
51
+ litgpt evaluate --tasks 'leaderboard' --out_dir 'evaluate-leaderboard/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
52
+
53
+ litgpt evaluate --tasks 'bbh_zeroshot,bbh_fewshot,bbh_cot_fewshot,bbh_cot_zeroshot' --out_dir 'evaluate-bigbenchhard/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
54
+
55
+ litgpt evaluate --tasks 'mmlu,mmlu_pro' --out_dir 'evaluate-mmlu/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
56
+
57
+ litgpt evaluate --tasks 'arc_challenge,boolq,gpqa,hellaswag,openbookqa,piqa,truthfulqa_mc2,winogrande' --out_dir 'evaluate-reasoning/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
58
+
59
+ litgpt evaluate --tasks 'mmlu_multilingual,mgsm' --out_dir 'evaluate-multilinguals/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
60
+
61
+ litgpt evaluate --tasks 'gsm8k,mathqa' --out_dir 'evaluate-math/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
62
+
63
+ litgpt evaluate --tasks 'qasper' --out_dir 'evaluate-long/' --batch_size 4 --dtype 'bfloat16' out/pretrain/final/
64
+ ```
scripts/prepare_contrain_dataset.py ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ # https://huggingface.co/datasets/Tongjilibo/self_cognition
3
+
4
+ https://huggingface.co/datasets/HuggingFaceH4/no_robots
5
+ https://huggingface.co/datasets/HuggingFaceH4/ultrachat_200k
6
+ https://huggingface.co/datasets/NousResearch/hermes-function-calling-v1
7
+ https://huggingface.co/datasets/Locutusque/function-calling-chatml
8
+ https://huggingface.co/datasets/cognitivecomputations/SystemChat-2.0
9
+
10
+ https://huggingface.co/datasets/teknium/OpenHermes-2.5
11
+
12
+ https://huggingface.co/datasets/cognitivecomputations/open-instruct-uncensored
13
+
14
+ https://huggingface.co/datasets/WizardLMTeam/WizardLM_evol_instruct_V2_196k
15
+ https://huggingface.co/datasets/HuggingFaceH4/deita-10k-v0-sft?row=0
16
+ https://huggingface.co/datasets/Open-Orca/slimorca-deduped-cleaned-corrected
17
+ https://huggingface.co/datasets/Undi95/andrijdavid_roleplay-conversation-sharegpt
18
+ https://huggingface.co/datasets/roleplay4fun/CoupleRP
19
+ https://huggingface.co/datasets/arcee-ai/EvolKit-20k
20
+ https://huggingface.co/datasets/arcee-ai/The-Tome
21
+ https://huggingface.co/datasets/arcee-ai/agent-data
22
+ https://huggingface.co/datasets/arcee-ai/reasoning-sharegpt
23
+ https://huggingface.co/datasets/arcee-ai/infini-instruct-top-500k
24
+ https://huggingface.co/datasets/arcee-ai/BAAI-Infinity-Instruct-System
25
+ # https://huggingface.co/datasets/arcee-ai/financial-instructions-cleaned-2
26
+
27
+ https://huggingface.co/datasets/KingNish/reasoning-base-20k
28
+ https://huggingface.co/datasets/Magpie-Align/Magpie-Reasoning-150K
29
+ https://huggingface.co/datasets/ai2-adapt-dev/openmath-2-math
30
+ https://huggingface.co/datasets/thesven/gsm8k-reasoning
31
+ """
scripts/prepare_pretrain_dataset.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+ from functools import partial
3
+
4
+ from datasets import load_dataset
5
+ from litdata import optimize, TokensLoader
6
+ from litgpt.tokenizer import Tokenizer
7
+
8
+
9
+ def batch_iterator(path: str,
10
+ name: Optional[str]=None,
11
+ data_dir: Optional[str]=None,
12
+ data_files: Optional[str]=None,
13
+ revision: Optional[str]=None,
14
+ split: str='train',
15
+ format: Optional[str]=None):
16
+ assert format is not None
17
+
18
+ dataset = load_dataset(path=path,
19
+ name=name,
20
+ data_dir=data_dir,
21
+ data_files=data_files,
22
+ revision=revision,
23
+ split=split,
24
+ trust_remote_code=True)
25
+
26
+ for row in dataset:
27
+ text = format.format(**row)
28
+ yield text
29
+
30
+
31
+ def tokenize_fn(datasets_config, tokenizer=None):
32
+ for text in batch_iterator(**datasets_config):
33
+ text_ids = tokenizer.encode(text, bos=False, eos=True)
34
+ yield text_ids
35
+
36
+
37
+ datasets_configs = [
38
+ # instruct
39
+ {'path': 'yahma/alpaca-cleaned', 'format': '{instruction} {input} {output}'},
40
+ {'path': 'gbharti/wealth-alpaca_lora', 'format': '{instruction} {input} {output}'},
41
+ {'path': 'databricks/databricks-dolly-15k', 'format': '{instruction} {context} {response}'},
42
+ {'path': 'VMware/open-instruct', 'format': '{instruction} {response}'},
43
+
44
+ # multilingual
45
+ *[
46
+ {'path': 'saillab/taco-datasets', 'data_dir': data_dir, 'split': 'train[:10%]', 'format': '{instruction} {input} {output}'}
47
+ for data_dir in [
48
+ 'multilingual-instruction-tuning-dataset /multilingual-alpaca-52k-gpt-4',
49
+ 'multilingual-instruction-tuning-dataset /multilinugal-dolly-15k',
50
+ ]
51
+ ],
52
+ *[
53
+ {'path': 'xu-song/cc100-samples', 'name': name, 'split': 'train[:10%]', 'format': '{text}'}
54
+ for name in [
55
+ 'am', 'ar', 'as', 'az', 'be', 'bg', 'bn', 'bn_rom', 'br',
56
+ 'bs', 'ca', 'cs', 'cy', 'da', 'de', 'el', 'en', 'eo', 'es',
57
+ 'et', 'eu', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl',
58
+ 'gn', 'gu', 'ha', 'he', 'hi', 'hi_rom', 'hr', 'ht', 'hu',
59
+ 'hy', 'id', 'ig', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km',
60
+ 'kn', 'ko', 'ku', 'ky', 'la', 'lg', 'li', 'ln', 'lo', 'lt',
61
+ 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'my_zaw',
62
+ 'ne', 'nl', 'no', 'ns', 'om', 'or', 'pa', 'pl', 'ps', 'pt',
63
+ 'qu', 'rm', 'ro', 'ru', 'sa', 'si', 'sc', 'sd', 'sk', 'sl',
64
+ 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'ta_rom',
65
+ 'te', 'te_rom', 'th', 'tl', 'tn', 'tr', 'ug', 'uk', 'ur',
66
+ 'ur_rom', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo',
67
+ 'zh-Hans', 'zh-Hant', 'zu',
68
+ ]
69
+ ],
70
+ # *[
71
+ # {'path': 'Salesforce/wikitext', 'name': name, 'split': 'train+validation+test', 'format': '{text}'}
72
+ # for name in [
73
+ # 'wikitext-103-raw-v1',
74
+ # 'wikitext-103-v1',
75
+ # 'wikitext-2-raw-v1',
76
+ # 'wikitext-2-v1',
77
+ # ]
78
+ # ],
79
+ {'path': 'jordiclive/wikipedia-summary-dataset', 'format': '{summary}'},
80
+ # {'path': 'ontocord/fineweb-permissive-multilingual-2m', 'split': 'train[:5%]', 'format': '{text}'},
81
+
82
+ # general
83
+ # {'path': 'MuskumPillerum/General-Knowledge', 'format': '{Question} {Answer}'},
84
+ # {'path': 'yirenc/general_knowledge_boolean', 'split': 'train+validation', 'format': '{question}? {answer}. {passage}'},
85
+ # {'path': 'nuvocare/MSD_instruct', 'split': 'train+test', 'format': '{Question} {Text}'},
86
+ # {'path': 'keivalya/MedQuad-MedicalQnADataset', 'split': 'train', 'format': '{Question} {Answer}'},
87
+ # {'path': 'NousResearch/CharacterCodex', 'split': 'train', 'format': '{scenario} {description}'},
88
+ # {'path': 'nampdn-ai/tiny-textbooks', 'split': 'train+test', 'format': '{textbook}'},
89
+
90
+ # code
91
+ {'path': 'nampdn-ai/tiny-codes', 'split': 'train[:5%]', 'format': '{prompt} {response}'},
92
+ *[
93
+ {'path': 'bigcode/the-stack-smol-xs', 'name': name, 'format': '{content}'}
94
+ for name in [
95
+ 'ada', 'agda', 'alloy', 'antlr', 'applescript', 'assembly',
96
+ 'augeas', 'awk', 'batchfile', 'bison', 'bluespec', 'c',
97
+ 'c++', 'c-sharp', 'clojure', 'cmake', 'coffeescript', 'common-lisp',
98
+ 'css', 'cuda', 'dart', 'dockerfile', 'elixir',
99
+ 'elm', 'emacs-lisp','erlang', 'f-sharp', 'fortran', 'glsl', 'go',
100
+ 'groovy', 'haskell','html', 'idris', 'isabelle', 'java',
101
+ 'java-server-pages', 'javascript', 'julia', 'kotlin', 'lean',
102
+ 'literate-agda', 'literate-coffeescript', 'literate-haskell',
103
+ 'lua', 'makefile', 'maple', 'markdown', 'mathematica', 'matlab',
104
+ 'ocaml', 'pascal', 'perl', 'php', 'powershell', 'prolog',
105
+ 'protocol-buffer', 'python', 'r', 'racket', 'restructuredtext',
106
+ 'rmarkdown', 'ruby', 'rust', 'sas', 'scala', 'scheme',
107
+ 'shell', 'smalltalk', 'solidity', 'sparql', 'sql', 'stan',
108
+ 'standard-ml', 'stata', 'systemverilog', 'tcl', 'tcsh', 'tex',
109
+ 'thrift', 'typescript', 'verilog', 'vhdl', 'visual-basic', 'xslt',
110
+ 'yacc', 'zig',
111
+ ]
112
+ ],
113
+ {'path': 'm-a-p/CodeFeedback-Filtered-Instruction', 'split': 'train', 'format': '{query} {answer}'},
114
+ {'path': 'jtatman/python-code-dataset-500k', 'format': '{instruction} {output}'},
115
+ {'path': 'iamtarun/python_code_instructions_18k_alpaca', 'format': '{instruction} {input} {output}'},
116
+ {'path': 'HuggingFaceH4/CodeAlpaca_20K', 'split': 'train+test', 'format': '{prompt} {completion}'},
117
+ {'path': 'cognitivecomputations/dolphin-coder', 'split': 'train', 'format': '{question} {response}'},
118
+
119
+ # math
120
+ {'path': 'fblgit/simple-math', 'revision': 'refs/convert/parquet', 'split': 'train+test', 'format': '{instruction} = {output}'},
121
+ {'path': 'gair-prox/open-web-math-pro', 'split': 'train[:5%]', 'format': '{text}'},
122
+ {'path': 'rvv-karma/Math-QA', 'split': 'train+val+test', 'format': '{question} {answer}'},
123
+ {'path': 'ajibawa-2023/Maths-College', 'split': 'train', 'format': '{instruction} {output}'},
124
+ {'path': 'microsoft/orca-math-word-problems-200k', 'format': '{question} {answer}'},
125
+ {'path': 'meta-math/MetaMathQA', 'format': '{query} {response}'},
126
+ {'path': 'TIGER-Lab/MathInstruct', 'format': '{instruction} {output}'},
127
+ {'path': 'TIGER-Lab/WebInstructSub', 'format': '{question} {answer}'},
128
+
129
+ # reasoning
130
+ {'path': 'SkunkworksAI/reasoning-0.01', 'format': '{instruction} {reasoning} {output}'},
131
+ {'path': 'KingNish/reasoning-base-20k', 'format': '{user} {reasoning} {assistant}'},
132
+ {'path': 'Magpie-Align/Magpie-Reasoning-150K', 'format': '{instruction} {response}'},
133
+ {'path': 'thesven/gsm8k-reasoning', 'format': '{question} {generation} {answer} {short_answer}'},
134
+ {'path': 'AlgorithmicResearchGroup/math_reasoning_autoformalization_track', 'format': '{informal_statement} {informal_proof} {formal_proof}'},
135
+
136
+ # misc
137
+ {'path': 'badrex/llm-emoji-dataset', 'format': '{character} {unicode} {short description} {tags} {LLM description}'},
138
+ ]
139
+
140
+ outputs = optimize(
141
+ fn=partial(tokenize_fn, tokenizer=Tokenizer('..')),
142
+ inputs=datasets_configs,
143
+ output_dir='../pretrain-data/',
144
+ # Number of tokens to store by chunks. This is roughly 64MB of tokens per chunk.
145
+ chunk_size=(2049 * 8012),
146
+ num_workers=32,
147
+ )
scripts/pretrain-model.yaml ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # The name of the model to pretrain. Choose from names in ``litgpt.config``. Mutually exclusive with
2
+ # ``model_config``. (type: Optional[str], default: null)
3
+ model_name: "Llama-3.2-1B"
4
+
5
+ # A ``litgpt.Config`` object to define the model architecture. Mutually exclusive with
6
+ # ``model_config``. (type: Optional[Config], default: null)
7
+ model_config:
8
+ padded_vocab_size: 38400
9
+ vocab_size: 38400
10
+ block_size: 8192
11
+ n_layer: 5
12
+ n_head: 32
13
+ head_size: null
14
+ n_embd: 1024
15
+ n_query_groups: 8
16
+ rotary_percentage: 1.0
17
+ parallel_residual: false
18
+ bias: false
19
+ norm_class_name: "RMSNorm"
20
+ norm_eps: 1e-05
21
+ mlp_class_name: "LLaMAMLP"
22
+ intermediate_size: 4096
23
+ rope_base: 500000
24
+ # rope_adjustments:
25
+ # factor: 32.0
26
+ # low_freq_factor: 1.0
27
+ # high_freq_factor: 4.0
28
+ # original_max_seq_len: 8192
29
+
30
+ # Directory in which to save checkpoints and logs. If running in a Lightning Studio Job, look for it in
31
+ # /teamspace/jobs/<job-name>/share. (type: <class 'Path'>, default: out/pretrain)
32
+ out_dir: "../out/pretrain/"
33
+
34
+ # The precision to use for pretraining. Possible choices: "bf16-true", "bf16-mixed", "32-true". (type: Optional[str], default: null)
35
+ # precision: bf16-mixed
36
+ precision: bf16-true
37
+
38
+ # Optional path to a checkpoint directory to initialize the model from.
39
+ # Useful for continued pretraining. Mutually exclusive with ``resume``. (type: Optional[Path], default: null)
40
+ initial_checkpoint_dir:
41
+
42
+ # Path to a checkpoint directory to resume from in case training was interrupted, or ``True`` to resume
43
+ # from the latest checkpoint in ``out_dir``. An error will be raised if no checkpoint is found. Passing
44
+ # ``'auto'`` will resume from the latest checkpoint but not error if no checkpoint exists.
45
+ # (type: Union[bool, Literal["auto"], Path], default: False)
46
+ # resume: false
47
+ resume: "auto"
48
+
49
+ # Data-related arguments. If not provided, the default is ``litgpt.data.TinyLlama``.
50
+ data:
51
+ class_path: LitData
52
+
53
+ init_args:
54
+ data_path: "../pretrain-data/"
55
+ num_workers: 16
56
+
57
+ # Training-related arguments. See ``litgpt.args.TrainArgs`` for details
58
+ train:
59
+ # Number of optimizer steps between saving checkpoints (type: Optional[int], default: 1000)
60
+ save_interval: 200
61
+
62
+ # Number of iterations between logging calls (type: int, default: 1)
63
+ log_interval: 1
64
+
65
+ # Number of samples between optimizer steps across data-parallel ranks (type: int, default: 512)
66
+ global_batch_size: 512
67
+
68
+ # Number of samples per data-parallel rank (type: int, default: 4)
69
+ # micro_batch_size: 16
70
+ micro_batch_size: 4
71
+
72
+ # Number of iterations with learning rate warmup active (type: int, default: 2000)
73
+ lr_warmup_steps: 2000
74
+
75
+ # Number of epochs to train on (type: Optional[int], default: null)
76
+ epochs:
77
+
78
+ # Total number of tokens to train on (type: Optional[int], default: 3000000000000)
79
+ # max_tokens: 3000000000000
80
+ # max_tokens: 8159107755 # 796399 * 2049 * 5
81
+ max_tokens: 11422750857 # 796399 * 2049 * 7
82
+
83
+ # Limits the number of optimizer steps to run. (type: Optional[int], default: null)
84
+ max_steps:
85
+
86
+ # Limits the length of samples. Off by default (type: Optional[int], default: null)
87
+ max_seq_length:
88
+
89
+ # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
90
+ tie_embeddings:
91
+
92
+ # (type: Optional[float], default: 1.0)
93
+ max_norm: 1.0
94
+
95
+ # (type: float, default: 4e-05)
96
+ min_lr: 1e-4
97
+
98
+ # Evaluation-related arguments. See ``litgpt.args.EvalArgs`` for details
99
+ eval:
100
+ # Number of optimizer steps between evaluation calls (type: int, default: 1000)
101
+ interval: 100
102
+
103
+ # Number of tokens to generate (type: Optional[int], default: null)
104
+ max_new_tokens:
105
+
106
+ # Number of iterations (type: int, default: 100)
107
+ max_iters: 100
108
+
109
+ # Whether to evaluate on the validation set at the beginning of the training
110
+ initial_validation: false
111
+
112
+ # Whether to evaluate on the validation set at the end the training
113
+ final_validation: true
114
+
115
+ # Optimizer-related arguments
116
+ optimizer:
117
+ # class_path: torch.optim.AdamW
118
+ class_path: grokadamw.GrokAdamW
119
+ # class_path: bitsandbytes.optim.AdamW8bit
120
+ # class_path: bitsandbytes.optim.PagedAdamW8bit
121
+
122
+ init_args:
123
+ # (type: float, default: 0.001)
124
+ # lr: 1e-3
125
+ lr: 1e-4
126
+
127
+ # (type: float, default: 0.01)
128
+ # weight_decay: 0.01
129
+ weight_decay: 0.1
130
+
131
+ # (type: tuple, default: (0.9,0.999))
132
+ betas:
133
+ - 0.9
134
+ - 0.95
135
+
136
+ # How many devices/GPUs to use. Uses all GPUs by default. (type: Union[int, str], default: auto)
137
+ devices: auto
138
+
139
+ # How many nodes to use. (type: int, default: 1)
140
+ num_nodes: 1
141
+
142
+ # Optional path to the tokenizer dir that was used for preprocessing the dataset. Only some data
143
+ # module require this. (type: Optional[Path], default: null)
144
+ tokenizer_dir: "../"
145
+
146
+ # The name of the logger to send metrics to. (type: Literal['wandb', 'tensorboard', 'csv'], default: tensorboard)
147
+ logger_name: "wandb"
148
+
149
+ # The random seed to use for reproducibility. (type: int, default: 42)
150
+ seed: 42
scripts/requirements.in ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
2
+
3
+ tqdm
4
+ datasets
5
+ jinja2
6
+ transformers
7
+ wandb
8
+ # litgpt[all]
9
+ litgpt[all] @ git+https://github.com/Lightning-AI/litgpt.git
10
+ litdata
11
+ grokadamw
12
+ # bitsandbytes
scripts/train_tokenizer.py ADDED
@@ -0,0 +1,337 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gc
2
+ import sys
3
+
4
+ from datasets import load_dataset
5
+ from transformers import PreTrainedTokenizerFast
6
+ from tokenizers import Tokenizer, normalizers, pre_tokenizers, processors, decoders
7
+ from tokenizers.models import BPE
8
+ from tokenizers.trainers import BpeTrainer
9
+ from tokenizers.processors import TemplateProcessing
10
+
11
+
12
+ x = input('Are you sure? [y/N] ')
13
+
14
+ if x not in ('y', 'Y', 'yes'):
15
+ sys.exit(0)
16
+
17
+
18
+ def batch_iterator():
19
+ # text
20
+ dataset = (
21
+ load_dataset('saillab/taco-datasets', data_dir=data_dir, split='train')
22
+ for data_dir in [
23
+ 'multilingual-instruction-tuning-dataset /multilingual-alpaca-52k-gpt-4',
24
+ 'multilingual-instruction-tuning-dataset /multilinugal-dolly-15k',
25
+ ]
26
+ )
27
+
28
+ for d in dataset:
29
+ for row in d:
30
+ for n in row:
31
+ yield row['instruction'] + '\n' + row['input'] + '\n' + row['output']
32
+
33
+ del dataset
34
+ gc.collect()
35
+
36
+ # text
37
+ dataset = (
38
+ load_dataset('xu-song/cc100-samples', lang, split='train')
39
+ for lang in [
40
+ 'am', 'ar', 'as', 'az', 'be', 'bg', 'bn', 'bn_rom', 'br',
41
+ 'bs', 'ca', 'cs', 'cy', 'da', 'de', 'el', 'en', 'eo', 'es',
42
+ 'et', 'eu', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl',
43
+ 'gn', 'gu', 'ha', 'he', 'hi', 'hi_rom', 'hr', 'ht', 'hu',
44
+ 'hy', 'id', 'ig', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km',
45
+ 'kn', 'ko', 'ku', 'ky', 'la', 'lg', 'li', 'ln', 'lo', 'lt',
46
+ 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'my_zaw',
47
+ 'ne', 'nl', 'no', 'ns', 'om', 'or', 'pa', 'pl', 'ps', 'pt',
48
+ 'qu', 'rm', 'ro', 'ru', 'sa', 'si', 'sc', 'sd', 'sk', 'sl',
49
+ 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'ta_rom',
50
+ 'te', 'te_rom', 'th', 'tl', 'tn', 'tr', 'ug', 'uk', 'ur',
51
+ 'ur_rom', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo',
52
+ 'zh-Hans', 'zh-Hant', 'zu',
53
+ ]
54
+ )
55
+
56
+ for d in dataset:
57
+ for row in d['text']:
58
+ yield row
59
+
60
+ del dataset
61
+ gc.collect()
62
+
63
+ # code
64
+ dataset = load_dataset('bigcode/programming-languages-keywords', split='train')
65
+
66
+ for row in dataset:
67
+ for n in row['keywords']:
68
+ yield n
69
+
70
+ del dataset
71
+ gc.collect()
72
+
73
+ # code
74
+ dataset = (
75
+ load_dataset('bigcode/the-stack-smol-xs', lang, split='train', trust_remote_code=True)
76
+ for lang in [
77
+ 'ada', 'agda', 'alloy', 'antlr', 'applescript', 'assembly',
78
+ 'augeas', 'awk', 'batchfile', 'bison', 'bluespec', 'c',
79
+ 'c++', 'c-sharp', 'clojure', 'cmake', 'coffeescript', 'common-lisp',
80
+ 'css', 'cuda', 'dart', 'dockerfile', 'elixir',
81
+ 'elm', 'emacs-lisp','erlang', 'f-sharp', 'fortran', 'glsl', 'go',
82
+ 'groovy', 'haskell','html', 'idris', 'isabelle', 'java',
83
+ 'java-server-pages', 'javascript', 'julia', 'kotlin', 'lean',
84
+ 'literate-agda', 'literate-coffeescript', 'literate-haskell',
85
+ 'lua', 'makefile', 'maple', 'markdown', 'mathematica', 'matlab',
86
+ 'ocaml', 'pascal', 'perl', 'php', 'powershell', 'prolog',
87
+ 'protocol-buffer', 'python', 'r', 'racket', 'restructuredtext',
88
+ 'rmarkdown', 'ruby', 'rust', 'sas', 'scala', 'scheme',
89
+ 'shell', 'smalltalk', 'solidity', 'sparql', 'sql', 'stan',
90
+ 'standard-ml', 'stata', 'systemverilog', 'tcl', 'tcsh', 'tex',
91
+ 'thrift', 'typescript', 'verilog', 'vhdl', 'visual-basic', 'xslt',
92
+ 'yacc', 'zig',
93
+ ]
94
+ )
95
+
96
+ for d in dataset:
97
+ for row in d:
98
+ yield row['content']
99
+
100
+ del dataset
101
+ gc.collect()
102
+
103
+ # text + code
104
+ dataset = load_dataset('m-a-p/CodeFeedback-Filtered-Instruction', split='train')
105
+
106
+ for row in dataset:
107
+ yield row['query'] + '\n' + row['answer']
108
+
109
+ del dataset
110
+ gc.collect()
111
+
112
+ # math
113
+ dataset = load_dataset('gair-prox/open-web-math-pro', split='train')
114
+
115
+ for row in dataset:
116
+ yield row['text']
117
+
118
+ del dataset
119
+ gc.collect()
120
+
121
+ # math
122
+ dataset = load_dataset('ajibawa-2023/Maths-College', split='train')
123
+
124
+ for row in dataset:
125
+ yield row['instruction'] + '\n' + row['output']
126
+
127
+ del dataset
128
+ gc.collect()
129
+
130
+ # math
131
+ dataset = load_dataset('microsoft/orca-math-word-problems-200k', split='train')
132
+
133
+ for row in dataset:
134
+ yield row['question'] + '\n' + row['answer']
135
+
136
+ del dataset
137
+ gc.collect()
138
+
139
+ # emoji
140
+ dataset = load_dataset('badrex/llm-emoji-dataset', split='train')
141
+
142
+ for row in dataset:
143
+ yield f'{row["character"]}\n{row["unicode"]}\n{row["short description"]}\n{row["tags"]}\n{row["LLM description"]}'
144
+
145
+ del dataset
146
+ gc.collect()
147
+
148
+
149
+ bpe = BPE(unk_token=None, fuse_unk=False, byte_fallback=False, ignore_merges=True)
150
+ tokenizer = Tokenizer(bpe)
151
+
152
+ special_tokens = [
153
+ '<unk>',
154
+ '<s>',
155
+ '</s>',
156
+ '<|im_start|>',
157
+ '<|im_end|>',
158
+ 'system',
159
+ 'user',
160
+ 'assistant',
161
+ 'resource',
162
+ 'tool',
163
+ 'agent',
164
+
165
+ # tool/function calling
166
+ '<tools>',
167
+ '</tools>',
168
+ '<tool_call>',
169
+ '</tool_call>',
170
+ '<tool_response>',
171
+ '</tool_response>',
172
+
173
+ '"arguments"',
174
+ '"name"',
175
+
176
+ '<arguments>',
177
+ '</arguments>',
178
+ '<argument>',
179
+ '</argument>',
180
+ '<argument-name>',
181
+ '</argument-name>',
182
+ '<argument-type>',
183
+ '</argument-type>',
184
+ '<argument-value>',
185
+ '</argument-value>',
186
+ '<parameter>',
187
+ '</parameter>',
188
+ '<parameter-name>',
189
+ '</parameter-name>',
190
+ '<parameter-type>',
191
+ '</parameter-type>',
192
+ '<parameter-value>',
193
+ '</parameter-value>',
194
+ '<field>',
195
+ '</field>',
196
+ '<field-name>',
197
+ '</field-name>',
198
+ '<field-type>',
199
+ '</field-type>',
200
+ '<field-value>',
201
+ '</field-value>',
202
+ '<name>',
203
+ '</name>',
204
+ '<type>',
205
+ '</type>',
206
+ '<value>',
207
+ '</value>',
208
+ '<function>',
209
+ '</function>',
210
+ '<function-name>',
211
+ '</function-name>',
212
+ '<function-type>',
213
+ '</function-type>',
214
+ '<function-value>',
215
+ '</function-value>',
216
+
217
+ # qa
218
+ '<qa>',
219
+ '</qa>',
220
+ '<question>',
221
+ '</question>',
222
+ '<answer>',
223
+ '</answer>',
224
+
225
+ # cot, tot
226
+ '<cot>',
227
+ '</cot>',
228
+ '<tot>',
229
+ '</tot>',
230
+ '<input>',
231
+ '</input>',
232
+ '<output>',
233
+ '</output>',
234
+ '<thoughts>',
235
+ '</thoughts>',
236
+ '<thought>',
237
+ '</thought>',
238
+ '<plans>',
239
+ '</plans>',
240
+ '<plan>',
241
+ '</plan>',
242
+ '<votes>',
243
+ '</votes>',
244
+ '<vote>',
245
+ '</vote>',
246
+ '<passages>',
247
+ '</passages>',
248
+ '<passage>',
249
+ '</passage>',
250
+
251
+ # react
252
+ '<react>',
253
+ '</react>',
254
+ '<reasoning>',
255
+ '</reasoning>',
256
+ '<acting>',
257
+ '</acting>',
258
+ '<action>',
259
+ '</action>',
260
+ '<observation>',
261
+ '</observation>',
262
+ '<claim>',
263
+ '</claim>',
264
+
265
+ # reflection
266
+ '<thinking>',
267
+ '</thinking>',
268
+ '<step>',
269
+ '</step>',
270
+ '<reflection>',
271
+ '</reflection>',
272
+ '<output>',
273
+ '</output>',
274
+ ]
275
+
276
+ for i in range(2, 25):
277
+ special_tokens.append(' ' * i)
278
+
279
+ for i in range(128 - len(special_tokens)):
280
+ special_tokens.append(f'<|reserved_{i}|>')
281
+
282
+ # emoji
283
+ dataset = load_dataset('badrex/llm-emoji-dataset', split='train')
284
+ emoji_chars = [row['character'] for row in dataset if len(row['character']) == 1]
285
+ del dataset
286
+
287
+ # programming languages
288
+ dataset = load_dataset('Tanvir1337/programming-languages', split='train')
289
+ programming_languages = [n for row in dataset for n in row['text']]
290
+ del dataset
291
+
292
+ # programming languages keywords
293
+ dataset = load_dataset('bigcode/programming-languages-keywords', split='train')
294
+ code_keywords = [n for row in dataset for n in row['keywords']]
295
+ del dataset
296
+
297
+ tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False, trim_offsets=True, use_regex=True)
298
+
299
+ tokenizer.post_processor = TemplateProcessing(
300
+ single='$A:0', # $A represents the token, :0 specifies the type ID for single sequences
301
+ pair='$A:0 $B:1', # For pairs, we specify type IDs for both tokens
302
+ special_tokens=[],
303
+ )
304
+
305
+ tokenizer.decoder = decoders.ByteLevel(add_prefix_space=False, trim_offsets=True, use_regex=True)
306
+
307
+ trainer = BpeTrainer(
308
+ vocab_size=38400, # 32768 chars + 5034 emojis
309
+ min_frequency=2,
310
+ special_tokens=special_tokens,
311
+ initial_alphabet=emoji_chars + programming_languages + code_keywords,
312
+ )
313
+
314
+ tokenizer.train_from_iterator(batch_iterator(), trainer)
315
+ tokenizer.save('../tokenizer.json')
316
+ tokenizer.model.save('../')
317
+
318
+ CHATML_CHAT_TEMPLATE = (
319
+ "{% for message in messages %}"
320
+ "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}"
321
+ "{% endfor %}"
322
+ "{% if add_generation_prompt %}"
323
+ "{{ '<|im_start|>assistant\n' }}"
324
+ "{% endif %}"
325
+ )
326
+
327
+ fast_tokenizer = PreTrainedTokenizerFast(
328
+ tokenizer_object=tokenizer,
329
+ chat_template=CHATML_CHAT_TEMPLATE,
330
+ bos_token='<s>',
331
+ eos_token='</s>',
332
+ unk_token='<unk>',
333
+ pad_token='</s>',
334
+ clean_up_tokenization_spaces=False,
335
+ )
336
+
337
+ fast_tokenizer.save_pretrained('../')
special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "eos_token": "</s>",
4
+ "pad_token": "</s>",
5
+ "unk_token": "<unk>"
6
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,1052 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<unk>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<|im_start|>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "4": {
36
+ "content": "<|im_end|>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "5": {
44
+ "content": "system",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "6": {
52
+ "content": "user",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "7": {
60
+ "content": "assistant",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "8": {
68
+ "content": "resource",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "9": {
76
+ "content": "tool",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "10": {
84
+ "content": "agent",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ },
91
+ "11": {
92
+ "content": "<tools>",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "12": {
100
+ "content": "</tools>",
101
+ "lstrip": false,
102
+ "normalized": false,
103
+ "rstrip": false,
104
+ "single_word": false,
105
+ "special": true
106
+ },
107
+ "13": {
108
+ "content": "<tool_call>",
109
+ "lstrip": false,
110
+ "normalized": false,
111
+ "rstrip": false,
112
+ "single_word": false,
113
+ "special": true
114
+ },
115
+ "14": {
116
+ "content": "</tool_call>",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false,
121
+ "special": true
122
+ },
123
+ "15": {
124
+ "content": "<tool_response>",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "16": {
132
+ "content": "</tool_response>",
133
+ "lstrip": false,
134
+ "normalized": false,
135
+ "rstrip": false,
136
+ "single_word": false,
137
+ "special": true
138
+ },
139
+ "17": {
140
+ "content": "\"arguments\"",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false,
145
+ "special": true
146
+ },
147
+ "18": {
148
+ "content": "\"name\"",
149
+ "lstrip": false,
150
+ "normalized": false,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": true
154
+ },
155
+ "19": {
156
+ "content": "<arguments>",
157
+ "lstrip": false,
158
+ "normalized": false,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": true
162
+ },
163
+ "20": {
164
+ "content": "</arguments>",
165
+ "lstrip": false,
166
+ "normalized": false,
167
+ "rstrip": false,
168
+ "single_word": false,
169
+ "special": true
170
+ },
171
+ "21": {
172
+ "content": "<argument>",
173
+ "lstrip": false,
174
+ "normalized": false,
175
+ "rstrip": false,
176
+ "single_word": false,
177
+ "special": true
178
+ },
179
+ "22": {
180
+ "content": "</argument>",
181
+ "lstrip": false,
182
+ "normalized": false,
183
+ "rstrip": false,
184
+ "single_word": false,
185
+ "special": true
186
+ },
187
+ "23": {
188
+ "content": "<argument-name>",
189
+ "lstrip": false,
190
+ "normalized": false,
191
+ "rstrip": false,
192
+ "single_word": false,
193
+ "special": true
194
+ },
195
+ "24": {
196
+ "content": "</argument-name>",
197
+ "lstrip": false,
198
+ "normalized": false,
199
+ "rstrip": false,
200
+ "single_word": false,
201
+ "special": true
202
+ },
203
+ "25": {
204
+ "content": "<argument-type>",
205
+ "lstrip": false,
206
+ "normalized": false,
207
+ "rstrip": false,
208
+ "single_word": false,
209
+ "special": true
210
+ },
211
+ "26": {
212
+ "content": "</argument-type>",
213
+ "lstrip": false,
214
+ "normalized": false,
215
+ "rstrip": false,
216
+ "single_word": false,
217
+ "special": true
218
+ },
219
+ "27": {
220
+ "content": "<argument-value>",
221
+ "lstrip": false,
222
+ "normalized": false,
223
+ "rstrip": false,
224
+ "single_word": false,
225
+ "special": true
226
+ },
227
+ "28": {
228
+ "content": "</argument-value>",
229
+ "lstrip": false,
230
+ "normalized": false,
231
+ "rstrip": false,
232
+ "single_word": false,
233
+ "special": true
234
+ },
235
+ "29": {
236
+ "content": "<parameter>",
237
+ "lstrip": false,
238
+ "normalized": false,
239
+ "rstrip": false,
240
+ "single_word": false,
241
+ "special": true
242
+ },
243
+ "30": {
244
+ "content": "</parameter>",
245
+ "lstrip": false,
246
+ "normalized": false,
247
+ "rstrip": false,
248
+ "single_word": false,
249
+ "special": true
250
+ },
251
+ "31": {
252
+ "content": "<parameter-name>",
253
+ "lstrip": false,
254
+ "normalized": false,
255
+ "rstrip": false,
256
+ "single_word": false,
257
+ "special": true
258
+ },
259
+ "32": {
260
+ "content": "</parameter-name>",
261
+ "lstrip": false,
262
+ "normalized": false,
263
+ "rstrip": false,
264
+ "single_word": false,
265
+ "special": true
266
+ },
267
+ "33": {
268
+ "content": "<parameter-type>",
269
+ "lstrip": false,
270
+ "normalized": false,
271
+ "rstrip": false,
272
+ "single_word": false,
273
+ "special": true
274
+ },
275
+ "34": {
276
+ "content": "</parameter-type>",
277
+ "lstrip": false,
278
+ "normalized": false,
279
+ "rstrip": false,
280
+ "single_word": false,
281
+ "special": true
282
+ },
283
+ "35": {
284
+ "content": "<parameter-value>",
285
+ "lstrip": false,
286
+ "normalized": false,
287
+ "rstrip": false,
288
+ "single_word": false,
289
+ "special": true
290
+ },
291
+ "36": {
292
+ "content": "</parameter-value>",
293
+ "lstrip": false,
294
+ "normalized": false,
295
+ "rstrip": false,
296
+ "single_word": false,
297
+ "special": true
298
+ },
299
+ "37": {
300
+ "content": "<field>",
301
+ "lstrip": false,
302
+ "normalized": false,
303
+ "rstrip": false,
304
+ "single_word": false,
305
+ "special": true
306
+ },
307
+ "38": {
308
+ "content": "</field>",
309
+ "lstrip": false,
310
+ "normalized": false,
311
+ "rstrip": false,
312
+ "single_word": false,
313
+ "special": true
314
+ },
315
+ "39": {
316
+ "content": "<field-name>",
317
+ "lstrip": false,
318
+ "normalized": false,
319
+ "rstrip": false,
320
+ "single_word": false,
321
+ "special": true
322
+ },
323
+ "40": {
324
+ "content": "</field-name>",
325
+ "lstrip": false,
326
+ "normalized": false,
327
+ "rstrip": false,
328
+ "single_word": false,
329
+ "special": true
330
+ },
331
+ "41": {
332
+ "content": "<field-type>",
333
+ "lstrip": false,
334
+ "normalized": false,
335
+ "rstrip": false,
336
+ "single_word": false,
337
+ "special": true
338
+ },
339
+ "42": {
340
+ "content": "</field-type>",
341
+ "lstrip": false,
342
+ "normalized": false,
343
+ "rstrip": false,
344
+ "single_word": false,
345
+ "special": true
346
+ },
347
+ "43": {
348
+ "content": "<field-value>",
349
+ "lstrip": false,
350
+ "normalized": false,
351
+ "rstrip": false,
352
+ "single_word": false,
353
+ "special": true
354
+ },
355
+ "44": {
356
+ "content": "</field-value>",
357
+ "lstrip": false,
358
+ "normalized": false,
359
+ "rstrip": false,
360
+ "single_word": false,
361
+ "special": true
362
+ },
363
+ "45": {
364
+ "content": "<name>",
365
+ "lstrip": false,
366
+ "normalized": false,
367
+ "rstrip": false,
368
+ "single_word": false,
369
+ "special": true
370
+ },
371
+ "46": {
372
+ "content": "</name>",
373
+ "lstrip": false,
374
+ "normalized": false,
375
+ "rstrip": false,
376
+ "single_word": false,
377
+ "special": true
378
+ },
379
+ "47": {
380
+ "content": "<type>",
381
+ "lstrip": false,
382
+ "normalized": false,
383
+ "rstrip": false,
384
+ "single_word": false,
385
+ "special": true
386
+ },
387
+ "48": {
388
+ "content": "</type>",
389
+ "lstrip": false,
390
+ "normalized": false,
391
+ "rstrip": false,
392
+ "single_word": false,
393
+ "special": true
394
+ },
395
+ "49": {
396
+ "content": "<value>",
397
+ "lstrip": false,
398
+ "normalized": false,
399
+ "rstrip": false,
400
+ "single_word": false,
401
+ "special": true
402
+ },
403
+ "50": {
404
+ "content": "</value>",
405
+ "lstrip": false,
406
+ "normalized": false,
407
+ "rstrip": false,
408
+ "single_word": false,
409
+ "special": true
410
+ },
411
+ "51": {
412
+ "content": "<function>",
413
+ "lstrip": false,
414
+ "normalized": false,
415
+ "rstrip": false,
416
+ "single_word": false,
417
+ "special": true
418
+ },
419
+ "52": {
420
+ "content": "</function>",
421
+ "lstrip": false,
422
+ "normalized": false,
423
+ "rstrip": false,
424
+ "single_word": false,
425
+ "special": true
426
+ },
427
+ "53": {
428
+ "content": "<function-name>",
429
+ "lstrip": false,
430
+ "normalized": false,
431
+ "rstrip": false,
432
+ "single_word": false,
433
+ "special": true
434
+ },
435
+ "54": {
436
+ "content": "</function-name>",
437
+ "lstrip": false,
438
+ "normalized": false,
439
+ "rstrip": false,
440
+ "single_word": false,
441
+ "special": true
442
+ },
443
+ "55": {
444
+ "content": "<function-type>",
445
+ "lstrip": false,
446
+ "normalized": false,
447
+ "rstrip": false,
448
+ "single_word": false,
449
+ "special": true
450
+ },
451
+ "56": {
452
+ "content": "</function-type>",
453
+ "lstrip": false,
454
+ "normalized": false,
455
+ "rstrip": false,
456
+ "single_word": false,
457
+ "special": true
458
+ },
459
+ "57": {
460
+ "content": "<function-value>",
461
+ "lstrip": false,
462
+ "normalized": false,
463
+ "rstrip": false,
464
+ "single_word": false,
465
+ "special": true
466
+ },
467
+ "58": {
468
+ "content": "</function-value>",
469
+ "lstrip": false,
470
+ "normalized": false,
471
+ "rstrip": false,
472
+ "single_word": false,
473
+ "special": true
474
+ },
475
+ "59": {
476
+ "content": "<qa>",
477
+ "lstrip": false,
478
+ "normalized": false,
479
+ "rstrip": false,
480
+ "single_word": false,
481
+ "special": true
482
+ },
483
+ "60": {
484
+ "content": "</qa>",
485
+ "lstrip": false,
486
+ "normalized": false,
487
+ "rstrip": false,
488
+ "single_word": false,
489
+ "special": true
490
+ },
491
+ "61": {
492
+ "content": "<question>",
493
+ "lstrip": false,
494
+ "normalized": false,
495
+ "rstrip": false,
496
+ "single_word": false,
497
+ "special": true
498
+ },
499
+ "62": {
500
+ "content": "</question>",
501
+ "lstrip": false,
502
+ "normalized": false,
503
+ "rstrip": false,
504
+ "single_word": false,
505
+ "special": true
506
+ },
507
+ "63": {
508
+ "content": "<answer>",
509
+ "lstrip": false,
510
+ "normalized": false,
511
+ "rstrip": false,
512
+ "single_word": false,
513
+ "special": true
514
+ },
515
+ "64": {
516
+ "content": "</answer>",
517
+ "lstrip": false,
518
+ "normalized": false,
519
+ "rstrip": false,
520
+ "single_word": false,
521
+ "special": true
522
+ },
523
+ "65": {
524
+ "content": "<cot>",
525
+ "lstrip": false,
526
+ "normalized": false,
527
+ "rstrip": false,
528
+ "single_word": false,
529
+ "special": true
530
+ },
531
+ "66": {
532
+ "content": "</cot>",
533
+ "lstrip": false,
534
+ "normalized": false,
535
+ "rstrip": false,
536
+ "single_word": false,
537
+ "special": true
538
+ },
539
+ "67": {
540
+ "content": "<tot>",
541
+ "lstrip": false,
542
+ "normalized": false,
543
+ "rstrip": false,
544
+ "single_word": false,
545
+ "special": true
546
+ },
547
+ "68": {
548
+ "content": "</tot>",
549
+ "lstrip": false,
550
+ "normalized": false,
551
+ "rstrip": false,
552
+ "single_word": false,
553
+ "special": true
554
+ },
555
+ "69": {
556
+ "content": "<input>",
557
+ "lstrip": false,
558
+ "normalized": false,
559
+ "rstrip": false,
560
+ "single_word": false,
561
+ "special": true
562
+ },
563
+ "70": {
564
+ "content": "</input>",
565
+ "lstrip": false,
566
+ "normalized": false,
567
+ "rstrip": false,
568
+ "single_word": false,
569
+ "special": true
570
+ },
571
+ "71": {
572
+ "content": "<output>",
573
+ "lstrip": false,
574
+ "normalized": false,
575
+ "rstrip": false,
576
+ "single_word": false,
577
+ "special": true
578
+ },
579
+ "72": {
580
+ "content": "</output>",
581
+ "lstrip": false,
582
+ "normalized": false,
583
+ "rstrip": false,
584
+ "single_word": false,
585
+ "special": true
586
+ },
587
+ "73": {
588
+ "content": "<thoughts>",
589
+ "lstrip": false,
590
+ "normalized": false,
591
+ "rstrip": false,
592
+ "single_word": false,
593
+ "special": true
594
+ },
595
+ "74": {
596
+ "content": "</thoughts>",
597
+ "lstrip": false,
598
+ "normalized": false,
599
+ "rstrip": false,
600
+ "single_word": false,
601
+ "special": true
602
+ },
603
+ "75": {
604
+ "content": "<thought>",
605
+ "lstrip": false,
606
+ "normalized": false,
607
+ "rstrip": false,
608
+ "single_word": false,
609
+ "special": true
610
+ },
611
+ "76": {
612
+ "content": "</thought>",
613
+ "lstrip": false,
614
+ "normalized": false,
615
+ "rstrip": false,
616
+ "single_word": false,
617
+ "special": true
618
+ },
619
+ "77": {
620
+ "content": "<plans>",
621
+ "lstrip": false,
622
+ "normalized": false,
623
+ "rstrip": false,
624
+ "single_word": false,
625
+ "special": true
626
+ },
627
+ "78": {
628
+ "content": "</plans>",
629
+ "lstrip": false,
630
+ "normalized": false,
631
+ "rstrip": false,
632
+ "single_word": false,
633
+ "special": true
634
+ },
635
+ "79": {
636
+ "content": "<plan>",
637
+ "lstrip": false,
638
+ "normalized": false,
639
+ "rstrip": false,
640
+ "single_word": false,
641
+ "special": true
642
+ },
643
+ "80": {
644
+ "content": "</plan>",
645
+ "lstrip": false,
646
+ "normalized": false,
647
+ "rstrip": false,
648
+ "single_word": false,
649
+ "special": true
650
+ },
651
+ "81": {
652
+ "content": "<votes>",
653
+ "lstrip": false,
654
+ "normalized": false,
655
+ "rstrip": false,
656
+ "single_word": false,
657
+ "special": true
658
+ },
659
+ "82": {
660
+ "content": "</votes>",
661
+ "lstrip": false,
662
+ "normalized": false,
663
+ "rstrip": false,
664
+ "single_word": false,
665
+ "special": true
666
+ },
667
+ "83": {
668
+ "content": "<vote>",
669
+ "lstrip": false,
670
+ "normalized": false,
671
+ "rstrip": false,
672
+ "single_word": false,
673
+ "special": true
674
+ },
675
+ "84": {
676
+ "content": "</vote>",
677
+ "lstrip": false,
678
+ "normalized": false,
679
+ "rstrip": false,
680
+ "single_word": false,
681
+ "special": true
682
+ },
683
+ "85": {
684
+ "content": "<passages>",
685
+ "lstrip": false,
686
+ "normalized": false,
687
+ "rstrip": false,
688
+ "single_word": false,
689
+ "special": true
690
+ },
691
+ "86": {
692
+ "content": "</passages>",
693
+ "lstrip": false,
694
+ "normalized": false,
695
+ "rstrip": false,
696
+ "single_word": false,
697
+ "special": true
698
+ },
699
+ "87": {
700
+ "content": "<passage>",
701
+ "lstrip": false,
702
+ "normalized": false,
703
+ "rstrip": false,
704
+ "single_word": false,
705
+ "special": true
706
+ },
707
+ "88": {
708
+ "content": "</passage>",
709
+ "lstrip": false,
710
+ "normalized": false,
711
+ "rstrip": false,
712
+ "single_word": false,
713
+ "special": true
714
+ },
715
+ "89": {
716
+ "content": "<react>",
717
+ "lstrip": false,
718
+ "normalized": false,
719
+ "rstrip": false,
720
+ "single_word": false,
721
+ "special": true
722
+ },
723
+ "90": {
724
+ "content": "</react>",
725
+ "lstrip": false,
726
+ "normalized": false,
727
+ "rstrip": false,
728
+ "single_word": false,
729
+ "special": true
730
+ },
731
+ "91": {
732
+ "content": "<reasoning>",
733
+ "lstrip": false,
734
+ "normalized": false,
735
+ "rstrip": false,
736
+ "single_word": false,
737
+ "special": true
738
+ },
739
+ "92": {
740
+ "content": "</reasoning>",
741
+ "lstrip": false,
742
+ "normalized": false,
743
+ "rstrip": false,
744
+ "single_word": false,
745
+ "special": true
746
+ },
747
+ "93": {
748
+ "content": "<acting>",
749
+ "lstrip": false,
750
+ "normalized": false,
751
+ "rstrip": false,
752
+ "single_word": false,
753
+ "special": true
754
+ },
755
+ "94": {
756
+ "content": "</acting>",
757
+ "lstrip": false,
758
+ "normalized": false,
759
+ "rstrip": false,
760
+ "single_word": false,
761
+ "special": true
762
+ },
763
+ "95": {
764
+ "content": "<action>",
765
+ "lstrip": false,
766
+ "normalized": false,
767
+ "rstrip": false,
768
+ "single_word": false,
769
+ "special": true
770
+ },
771
+ "96": {
772
+ "content": "</action>",
773
+ "lstrip": false,
774
+ "normalized": false,
775
+ "rstrip": false,
776
+ "single_word": false,
777
+ "special": true
778
+ },
779
+ "97": {
780
+ "content": "<observation>",
781
+ "lstrip": false,
782
+ "normalized": false,
783
+ "rstrip": false,
784
+ "single_word": false,
785
+ "special": true
786
+ },
787
+ "98": {
788
+ "content": "</observation>",
789
+ "lstrip": false,
790
+ "normalized": false,
791
+ "rstrip": false,
792
+ "single_word": false,
793
+ "special": true
794
+ },
795
+ "99": {
796
+ "content": "<claim>",
797
+ "lstrip": false,
798
+ "normalized": false,
799
+ "rstrip": false,
800
+ "single_word": false,
801
+ "special": true
802
+ },
803
+ "100": {
804
+ "content": "</claim>",
805
+ "lstrip": false,
806
+ "normalized": false,
807
+ "rstrip": false,
808
+ "single_word": false,
809
+ "special": true
810
+ },
811
+ "101": {
812
+ "content": "<thinking>",
813
+ "lstrip": false,
814
+ "normalized": false,
815
+ "rstrip": false,
816
+ "single_word": false,
817
+ "special": true
818
+ },
819
+ "102": {
820
+ "content": "</thinking>",
821
+ "lstrip": false,
822
+ "normalized": false,
823
+ "rstrip": false,
824
+ "single_word": false,
825
+ "special": true
826
+ },
827
+ "103": {
828
+ "content": "<step>",
829
+ "lstrip": false,
830
+ "normalized": false,
831
+ "rstrip": false,
832
+ "single_word": false,
833
+ "special": true
834
+ },
835
+ "104": {
836
+ "content": "</step>",
837
+ "lstrip": false,
838
+ "normalized": false,
839
+ "rstrip": false,
840
+ "single_word": false,
841
+ "special": true
842
+ },
843
+ "105": {
844
+ "content": "<reflection>",
845
+ "lstrip": false,
846
+ "normalized": false,
847
+ "rstrip": false,
848
+ "single_word": false,
849
+ "special": true
850
+ },
851
+ "106": {
852
+ "content": "</reflection>",
853
+ "lstrip": false,
854
+ "normalized": false,
855
+ "rstrip": false,
856
+ "single_word": false,
857
+ "special": true
858
+ },
859
+ "107": {
860
+ "content": " ",
861
+ "lstrip": false,
862
+ "normalized": false,
863
+ "rstrip": false,
864
+ "single_word": false,
865
+ "special": true
866
+ },
867
+ "108": {
868
+ "content": " ",
869
+ "lstrip": false,
870
+ "normalized": false,
871
+ "rstrip": false,
872
+ "single_word": false,
873
+ "special": true
874
+ },
875
+ "109": {
876
+ "content": " ",
877
+ "lstrip": false,
878
+ "normalized": false,
879
+ "rstrip": false,
880
+ "single_word": false,
881
+ "special": true
882
+ },
883
+ "110": {
884
+ "content": " ",
885
+ "lstrip": false,
886
+ "normalized": false,
887
+ "rstrip": false,
888
+ "single_word": false,
889
+ "special": true
890
+ },
891
+ "111": {
892
+ "content": " ",
893
+ "lstrip": false,
894
+ "normalized": false,
895
+ "rstrip": false,
896
+ "single_word": false,
897
+ "special": true
898
+ },
899
+ "112": {
900
+ "content": " ",
901
+ "lstrip": false,
902
+ "normalized": false,
903
+ "rstrip": false,
904
+ "single_word": false,
905
+ "special": true
906
+ },
907
+ "113": {
908
+ "content": " ",
909
+ "lstrip": false,
910
+ "normalized": false,
911
+ "rstrip": false,
912
+ "single_word": false,
913
+ "special": true
914
+ },
915
+ "114": {
916
+ "content": " ",
917
+ "lstrip": false,
918
+ "normalized": false,
919
+ "rstrip": false,
920
+ "single_word": false,
921
+ "special": true
922
+ },
923
+ "115": {
924
+ "content": " ",
925
+ "lstrip": false,
926
+ "normalized": false,
927
+ "rstrip": false,
928
+ "single_word": false,
929
+ "special": true
930
+ },
931
+ "116": {
932
+ "content": " ",
933
+ "lstrip": false,
934
+ "normalized": false,
935
+ "rstrip": false,
936
+ "single_word": false,
937
+ "special": true
938
+ },
939
+ "117": {
940
+ "content": " ",
941
+ "lstrip": false,
942
+ "normalized": false,
943
+ "rstrip": false,
944
+ "single_word": false,
945
+ "special": true
946
+ },
947
+ "118": {
948
+ "content": " ",
949
+ "lstrip": false,
950
+ "normalized": false,
951
+ "rstrip": false,
952
+ "single_word": false,
953
+ "special": true
954
+ },
955
+ "119": {
956
+ "content": " ",
957
+ "lstrip": false,
958
+ "normalized": false,
959
+ "rstrip": false,
960
+ "single_word": false,
961
+ "special": true
962
+ },
963
+ "120": {
964
+ "content": " ",
965
+ "lstrip": false,
966
+ "normalized": false,
967
+ "rstrip": false,
968
+ "single_word": false,
969
+ "special": true
970
+ },
971
+ "121": {
972
+ "content": " ",
973
+ "lstrip": false,
974
+ "normalized": false,
975
+ "rstrip": false,
976
+ "single_word": false,
977
+ "special": true
978
+ },
979
+ "122": {
980
+ "content": " ",
981
+ "lstrip": false,
982
+ "normalized": false,
983
+ "rstrip": false,
984
+ "single_word": false,
985
+ "special": true
986
+ },
987
+ "123": {
988
+ "content": " ",
989
+ "lstrip": false,
990
+ "normalized": false,
991
+ "rstrip": false,
992
+ "single_word": false,
993
+ "special": true
994
+ },
995
+ "124": {
996
+ "content": " ",
997
+ "lstrip": false,
998
+ "normalized": false,
999
+ "rstrip": false,
1000
+ "single_word": false,
1001
+ "special": true
1002
+ },
1003
+ "125": {
1004
+ "content": " ",
1005
+ "lstrip": false,
1006
+ "normalized": false,
1007
+ "rstrip": false,
1008
+ "single_word": false,
1009
+ "special": true
1010
+ },
1011
+ "126": {
1012
+ "content": " ",
1013
+ "lstrip": false,
1014
+ "normalized": false,
1015
+ "rstrip": false,
1016
+ "single_word": false,
1017
+ "special": true
1018
+ },
1019
+ "127": {
1020
+ "content": " ",
1021
+ "lstrip": false,
1022
+ "normalized": false,
1023
+ "rstrip": false,
1024
+ "single_word": false,
1025
+ "special": true
1026
+ },
1027
+ "128": {
1028
+ "content": " ",
1029
+ "lstrip": false,
1030
+ "normalized": false,
1031
+ "rstrip": false,
1032
+ "single_word": false,
1033
+ "special": true
1034
+ },
1035
+ "129": {
1036
+ "content": " ",
1037
+ "lstrip": false,
1038
+ "normalized": false,
1039
+ "rstrip": false,
1040
+ "single_word": false,
1041
+ "special": true
1042
+ }
1043
+ },
1044
+ "bos_token": "<s>",
1045
+ "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
1046
+ "clean_up_tokenization_spaces": false,
1047
+ "eos_token": "</s>",
1048
+ "model_max_length": 1000000000000000019884624838656,
1049
+ "pad_token": "</s>",
1050
+ "tokenizer_class": "PreTrainedTokenizerFast",
1051
+ "unk_token": "<unk>"
1052
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff