cowshed-base / added_tokens.json
ddobokki's picture
Upload tokenizer
5d3f074 verified
raw
history blame
1.23 kB
{
"<|bos|>": 50257,
"<|end_header_id|>": 50261,
"<|endoftext|>": 50256,
"<|eot_id|>": 50262,
"<|pad|>": 50258,
"<|start_header_id|>": 50260,
"<|unk|>": 50259,
"<|unused_10|>": 50272,
"<|unused_11|>": 50273,
"<|unused_12|>": 50274,
"<|unused_13|>": 50275,
"<|unused_14|>": 50276,
"<|unused_15|>": 50277,
"<|unused_16|>": 50278,
"<|unused_17|>": 50279,
"<|unused_18|>": 50280,
"<|unused_19|>": 50281,
"<|unused_1|>": 50263,
"<|unused_20|>": 50282,
"<|unused_21|>": 50283,
"<|unused_22|>": 50284,
"<|unused_23|>": 50285,
"<|unused_24|>": 50286,
"<|unused_25|>": 50287,
"<|unused_26|>": 50288,
"<|unused_27|>": 50289,
"<|unused_28|>": 50290,
"<|unused_29|>": 50291,
"<|unused_2|>": 50264,
"<|unused_30|>": 50292,
"<|unused_31|>": 50293,
"<|unused_32|>": 50294,
"<|unused_33|>": 50295,
"<|unused_34|>": 50296,
"<|unused_35|>": 50297,
"<|unused_36|>": 50298,
"<|unused_37|>": 50299,
"<|unused_38|>": 50300,
"<|unused_39|>": 50301,
"<|unused_3|>": 50265,
"<|unused_40|>": 50302,
"<|unused_41|>": 50303,
"<|unused_4|>": 50266,
"<|unused_5|>": 50267,
"<|unused_6|>": 50268,
"<|unused_7|>": 50269,
"<|unused_8|>": 50270,
"<|unused_9|>": 50271
}