relik-ie's picture
Upload tokenizer
66efc1a verified
{
"additional_special_tokens": [
"[R-0]",
"[R-1]",
"[R-2]",
"[R-3]",
"[R-4]",
"[R-5]",
"[R-6]",
"[R-7]",
"[R-8]",
"[R-9]",
"[R-10]",
"[R-11]",
"[R-12]",
"[R-13]",
"[R-14]",
"[R-15]",
"[R-16]",
"[R-17]",
"[R-18]",
"[R-19]",
"[R-20]",
"[R-21]",
"[R-22]",
"[R-23]",
"[R-24]",
"[R-25]",
"[R-26]",
"[R-27]",
"[R-28]",
"[R-29]",
"[R-30]",
"[R-31]",
"[R-32]",
"[R-33]",
"[R-34]",
"[R-35]",
"[R-36]",
"[R-37]",
"[R-38]",
"[R-39]",
"[R-40]",
"[R-41]",
"[R-42]",
"[R-43]",
"[R-44]",
"[R-45]",
"[R-46]",
"[R-47]",
"[R-48]",
"[R-49]"
],
"bos_token": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"cls_token": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"mask_token": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"sep_token": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "[UNK]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}