sanchit-gandhi's picture
Upload tokenizer
d9fa33e
raw
history blame
444 Bytes
{
" ": 37,
"b": 35,
"c": 36,
"m": 34,
"|": 0,
"ئ": 16,
"ا": 2,
"ب": 9,
"ت": 11,
"ج": 26,
"ح": 29,
"خ": 19,
"د": 8,
"ر": 7,
"ز": 22,
"س": 15,
"ش": 23,
"ع": 31,
"غ": 33,
"ف": 28,
"ق": 30,
"ك": 10,
"ل": 14,
"م": 12,
"ن": 4,
"ه": 13,
"و": 5,
"پ": 24,
"چ": 25,
"ڕ": 27,
"ژ": 21,
"ڤ": 18,
"گ": 20,
"ڵ": 32,
"ۆ": 17,
"ی": 3,
"ێ": 6,
"ە": 1
}