knc_arab_10mb / added_tokens.json
goldfish-models's picture
Upload knc_arab_10mb tokenizer.
71888fa
{"[XXXXX69]": 26541, "[XXXXX11]": 26483, "[XXXXX49]": 26521, "[XXXXX35]": 26507, "[XXXXX79]": 26551, "[XXXXX67]": 26539, "[XXXXX139]": 26611, "[XXXXX21]": 26493, "[XXXXX48]": 26520, "[XXXXX81]": 26553, "[XXXXX63]": 26535, "[XXXXX83]": 26555, "[XXXXX133]": 26605, "[XXXXX76]": 26548, "[XXXXX112]": 26584, "[CLS]": 26468, "[XXXXX60]": 26532, "[XXXXX131]": 26603, "[XXXXX126]": 26598, "[XXXXX37]": 26509, "[XXXXX46]": 26518, "[XXXXX62]": 26534, "[XXXXX106]": 26578, "[XXXXX58]": 26530, "[XXXXX72]": 26544, "[XXXXX130]": 26602, "[XXXXX100]": 26572, "[XXXXX74]": 26546, "[XXXXX18]": 26490, "[XXXXX140]": 26612, "[XXXXX121]": 26593, "[XXXXX111]": 26583, "[SEP]": 26469, "[XXXXX96]": 26568, "[XXXXX107]": 26579, "[XXXXX132]": 26604, "[XXXXX33]": 26505, "[XXXXX108]": 26580, "[XXXXX147]": 26619, "[XXXXX15]": 26487, "[XXXXX30]": 26502, "[XXXXX119]": 26591, "[XXXXX50]": 26522, "[XXXXX145]": 26617, "[XXXXX45]": 26517, "[XXXXX68]": 26540, "[XXXXX138]": 26610, "[XXXXX0]": 26472, "[XXXXX23]": 26495, "[XXXXX80]": 26552, "[XXXXX151]": 26623, "[XXXXX22]": 26494, "[XXXXX93]": 26565, "[XXXXX97]": 26569, "[XXXXX9]": 26481, "[XXXXX29]": 26501, "[XXXXX24]": 26496, "[XXXXX95]": 26567, "[XXXXX4]": 26476, "[XXXXX27]": 26499, "[XXXXX14]": 26486, "[XXXXX105]": 26577, "[XXXXX59]": 26531, "[XXXXX143]": 26615, "[XXXXX38]": 26510, "[XXXXX3]": 26475, "[XXXXX146]": 26618, "[XXXXX31]": 26503, "[XXXXX104]": 26576, "[XXXXX77]": 26549, "[XXXXX125]": 26597, "[XXXXX19]": 26491, "[XXXXX7]": 26479, "[XXXXX103]": 26575, "[XXXXX28]": 26500, "[XXXXX42]": 26514, "[XXXXX82]": 26554, "[XXXXX44]": 26516, "[XXXXX98]": 26570, "[XXXXX118]": 26590, "[XXXXX52]": 26524, "[XXXXX2]": 26474, "[XXXXX144]": 26616, "[XXXXX141]": 26613, "[MASK]": 26471, "[XXXXX136]": 26608, "[XXXXX99]": 26571, "[XXXXX53]": 26525, "[XXXXX110]": 26582, "[XXXXX116]": 26588, "[XXXXX12]": 26484, "[XXXXX117]": 26589, "[XXXXX91]": 26563, "[XXXXX36]": 26508, "[XXXXX85]": 26557, "[XXXXX56]": 26528, "[XXXXX73]": 26545, "[XXXXX101]": 26573, "[XXXXX75]": 26547, "[XXXXX92]": 26564, "[XXXXX20]": 26492, "[XXXXX6]": 26478, "[XXXXX113]": 26585, "[XXXXX65]": 26537, "[XXXXX26]": 26498, "[XXXXX109]": 26581, "[XXXXX70]": 26542, "[XXXXX40]": 26512, "[XXXXX78]": 26550, "[XXXXX86]": 26558, "[XXXXX134]": 26606, "[XXXXX123]": 26595, "[XXXXX39]": 26511, "[XXXXX84]": 26556, "[XXXXX89]": 26561, "[XXXXX10]": 26482, "[XXXXX137]": 26609, "[XXXXX64]": 26536, "[XXXXX142]": 26614, "[XXXXX148]": 26620, "[XXXXX114]": 26586, "[XXXXX55]": 26527, "[XXXXX94]": 26566, "[XXXXX88]": 26560, "[XXXXX66]": 26538, "[XXXXX71]": 26543, "[XXXXX32]": 26504, "[XXXXX129]": 26601, "[XXXXX13]": 26485, "[XXXXX34]": 26506, "[XXXXX102]": 26574, "[XXXXX47]": 26519, "[XXXXX61]": 26533, "[XXXXX135]": 26607, "<pad>": 26470, "[XXXXX120]": 26592, "[XXXXX17]": 26489, "[XXXXX127]": 26599, "[XXXXX90]": 26562, "[XXXXX5]": 26477, "[XXXXX51]": 26523, "[XXXXX57]": 26529, "[XXXXX122]": 26594, "[XXXXX41]": 26513, "[XXXXX25]": 26497, "[XXXXX1]": 26473, "[XXXXX124]": 26596, "[XXXXX87]": 26559, "[XXXXX54]": 26526, "[XXXXX8]": 26480, "[XXXXX149]": 26621, "[XXXXX150]": 26622, "[XXXXX43]": 26515, "[XXXXX115]": 26587, "[XXXXX128]": 26600, "[XXXXX16]": 26488}