skript-1m-gpt-neo125m / special_tokens_map.json
johnpaulbin's picture
add tokenizer
c50b71c
raw
history blame contribute delete
No virus
422 Bytes
{"bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "additional_special_tokens": ["\t\t", " ", " ", " "]}