relik-ie's picture
Upload tokenizer
12aad1c verified
raw
history blame
2.4 kB
{
"additional_special_tokens": [
"[R-0]",
"[R-1]",
"[R-2]",
"[R-3]",
"[R-4]",
"[R-5]",
"[R-6]",
"[R-7]",
"[R-8]",
"[R-9]",
"[R-10]",
"[R-11]",
"[R-12]",
"[R-13]",
"[R-14]",
"[R-15]",
"[R-16]",
"[R-17]",
"[R-18]",
"[R-19]",
"[R-20]",
"[R-21]",
"[R-22]",
"[R-23]",
"[R-24]",
"--NME--",
"[E-0]",
"[E-1]",
"[E-2]",
"[E-3]",
"[E-4]",
"[E-5]",
"[E-6]",
"[E-7]",
"[E-8]",
"[E-9]",
"[E-10]",
"[E-11]",
"[E-12]",
"[E-13]",
"[E-14]",
"[E-15]",
"[E-16]",
"[E-17]",
"[E-18]",
"[E-19]",
"[E-20]",
"[E-21]",
"[E-22]",
"[E-23]",
"[E-24]",
"[E-25]",
"[E-26]",
"[E-27]",
"[E-28]",
"[E-29]",
"[E-30]",
"[E-31]",
"[E-32]",
"[E-33]",
"[E-34]",
"[E-35]",
"[E-36]",
"[E-37]",
"[E-38]",
"[E-39]",
"[E-40]",
"[E-41]",
"[E-42]",
"[E-43]",
"[E-44]",
"[E-45]",
"[E-46]",
"[E-47]",
"[E-48]",
"[E-49]",
"[E-50]",
"[E-51]",
"[E-52]",
"[E-53]",
"[E-54]",
"[E-55]",
"[E-56]",
"[E-57]",
"[E-58]",
"[E-59]",
"[E-60]",
"[E-61]",
"[E-62]",
"[E-63]",
"[E-64]",
"[E-65]",
"[E-66]",
"[E-67]",
"[E-68]",
"[E-69]",
"[E-70]",
"[E-71]",
"[E-72]",
"[E-73]",
"[E-74]"
],
"bos_token": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"cls_token": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"mask_token": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"sep_token": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "[UNK]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}