Upload tokenizer
12aad1c
verified
|
{ |
|
"--NME--": 128026, |
|
"[E-0]": 128027, |
|
"[E-10]": 128037, |
|
"[E-11]": 128038, |
|
"[E-12]": 128039, |
|
"[E-13]": 128040, |
|
"[E-14]": 128041, |
|
"[E-15]": 128042, |
|
"[E-16]": 128043, |
|
"[E-17]": 128044, |
|
"[E-18]": 128045, |
|
"[E-19]": 128046, |
|
"[E-1]": 128028, |
|
"[E-20]": 128047, |
|
"[E-21]": 128048, |
|
"[E-22]": 128049, |
|
"[E-23]": 128050, |
|
"[E-24]": 128051, |
|
"[E-25]": 128052, |
|
"[E-26]": 128053, |
|
"[E-27]": 128054, |
|
"[E-28]": 128055, |
|
"[E-29]": 128056, |
|
"[E-2]": 128029, |
|
"[E-30]": 128057, |
|
"[E-31]": 128058, |
|
"[E-32]": 128059, |
|
"[E-33]": 128060, |
|
"[E-34]": 128061, |
|
"[E-35]": 128062, |
|
"[E-36]": 128063, |
|
"[E-37]": 128064, |
|
"[E-38]": 128065, |
|
"[E-39]": 128066, |
|
"[E-3]": 128030, |
|
"[E-40]": 128067, |
|
"[E-41]": 128068, |
|
"[E-42]": 128069, |
|
"[E-43]": 128070, |
|
"[E-44]": 128071, |
|
"[E-45]": 128072, |
|
"[E-46]": 128073, |
|
"[E-47]": 128074, |
|
"[E-48]": 128075, |
|
"[E-49]": 128076, |
|
"[E-4]": 128031, |
|
"[E-50]": 128077, |
|
"[E-51]": 128078, |
|
"[E-52]": 128079, |
|
"[E-53]": 128080, |
|
"[E-54]": 128081, |
|
"[E-55]": 128082, |
|
"[E-56]": 128083, |
|
"[E-57]": 128084, |
|
"[E-58]": 128085, |
|
"[E-59]": 128086, |
|
"[E-5]": 128032, |
|
"[E-60]": 128087, |
|
"[E-61]": 128088, |
|
"[E-62]": 128089, |
|
"[E-63]": 128090, |
|
"[E-64]": 128091, |
|
"[E-65]": 128092, |
|
"[E-66]": 128093, |
|
"[E-67]": 128094, |
|
"[E-68]": 128095, |
|
"[E-69]": 128096, |
|
"[E-6]": 128033, |
|
"[E-70]": 128097, |
|
"[E-71]": 128098, |
|
"[E-72]": 128099, |
|
"[E-73]": 128100, |
|
"[E-74]": 128101, |
|
"[E-7]": 128034, |
|
"[E-8]": 128035, |
|
"[E-9]": 128036, |
|
"[MASK]": 128000, |
|
"[R-0]": 128001, |
|
"[R-10]": 128011, |
|
"[R-11]": 128012, |
|
"[R-12]": 128013, |
|
"[R-13]": 128014, |
|
"[R-14]": 128015, |
|
"[R-15]": 128016, |
|
"[R-16]": 128017, |
|
"[R-17]": 128018, |
|
"[R-18]": 128019, |
|
"[R-19]": 128020, |
|
"[R-1]": 128002, |
|
"[R-20]": 128021, |
|
"[R-21]": 128022, |
|
"[R-22]": 128023, |
|
"[R-23]": 128024, |
|
"[R-24]": 128025, |
|
"[R-2]": 128003, |
|
"[R-3]": 128004, |
|
"[R-4]": 128005, |
|
"[R-5]": 128006, |
|
"[R-6]": 128007, |
|
"[R-7]": 128008, |
|
"[R-8]": 128009, |
|
"[R-9]": 128010 |
|
} |
|
|