add tokenizer

vocab.json

@@ -1 +1 @@
-{"ء": 1, "آ": 2, "أ": 3, "ؤ": 4, "إ": 5, "ئ": 6, "ا": 7, "ب": 8, "ة": 9, "ت": 10, "ث": 11, "ج": 12, "ح": 13, "خ": 14, "د": 15, "ذ": 16, "ر": 17, "ز": 18, "س": 19, "ش": 20, "ص": 21, "ض": 22, "ط": 23, "ظ": 24, "ع": 25, "غ": 26, "ف": 27, "ق": 28, "ك": 29, "ل": 30, "م": 31, "ن": 32, "ه": 33, "و": 34, "ى": 35, "ي": 36, "ٌ": 37, "ْ": 38, "چ": 39, "ڨ": 40, "ک": 41, "ھ": 42, "ی": 43, "ۘ": 44, "ﺃ": 45, "ﻻ": 46, "|": 0, "<unk>": 47, "<pad>": 48, "<s>": 49, "</s>": 50}
+{"ء": 1, "آ": 2, "أ": 3, "ؤ": 4, "إ": 5, "ئ": 6, "ا": 7, "ب": 8, "ة": 9, "ت": 10, "ث": 11, "ج": 12, "ح": 13, "خ": 14, "د": 15, "ذ": 16, "ر": 17, "ز": 18, "س": 19, "ش": 20, "ص": 21, "ض": 22, "ط": 23, "ظ": 24, "ع": 25, "غ": 26, "ف": 27, "ق": 28, "ك": 29, "ل": 30, "م": 31, "ن": 32, "ه": 33, "و": 34, "ى": 35, "ي": 36, "َ": 37, "ُ": 38, "ِ": 39, "ْ": 40, "چ": 41, "ڨ": 42, "ک": 43, "ھ": 44, "ی": 45, "ﺃ": 46, "ﻻ": 47, "|": 0, "<unk>": 48, "<pad>": 49, "<s>": 50, "</s>": 51}