Cafet's picture
Upload tokenizer
9269005 verified
raw
history blame
793 Bytes
{
"!": 1,
"\"": 2,
"%": 3,
"&": 4,
"'": 5,
"+": 6,
",": 7,
"-": 8,
".": 9,
"/": 10,
":": 11,
";": 12,
"?": 13,
"[": 14,
"[PAD]": 66,
"[UNK]": 65,
"]": 15,
"|": 0,
"~": 16,
" ": 17,
"«": 18,
"²": 19,
"»": 20,
"½": 21,
"¾": 22,
"õ": 23,
"а": 24,
"б": 25,
"в": 26,
"г": 27,
"д": 28,
"е": 29,
"ж": 30,
"з": 31,
"и": 32,
"й": 33,
"к": 34,
"л": 35,
"м": 36,
"н": 37,
"о": 38,
"п": 39,
"р": 40,
"с": 41,
"т": 42,
"у": 43,
"ф": 44,
"х": 45,
"ц": 46,
"ч": 47,
"ш": 48,
"щ": 49,
"ъ": 50,
"ы": 51,
"ь": 52,
"э": 53,
"ю": 54,
"я": 55,
"ё": 56,
"ү": 57,
"ө": 58,
"​": 59,
"–": 60,
"‘": 61,
"’": 62,
"“": 63,
"”": 64
}