fix: add default special tokens
Browse files- tokenizer_config.json +4 -1
tokenizer_config.json
CHANGED
@@ -5,5 +5,8 @@
|
|
5 |
"tokenization_arcade100k.Arcade100kTokenizer",
|
6 |
null
|
7 |
]
|
8 |
-
}
|
|
|
|
|
|
|
9 |
}
|
|
|
5 |
"tokenization_arcade100k.Arcade100kTokenizer",
|
6 |
null
|
7 |
]
|
8 |
+
},
|
9 |
+
"bos_token": "<|endoftext|>",
|
10 |
+
"eos_token": "<|endoftext|>",
|
11 |
+
"pad_token": "<|endoftext|>"
|
12 |
}
|