add tokenizer

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -0,0 +1 @@
+{"ँ": 1, "ं": 2, "ः": 3, "अ": 4, "आ": 5, "इ": 6, "ई": 7, "उ": 8, "ऊ": 9, "ऋ": 10, "ऍ": 11, "ए": 12, "ऐ": 13, "ऑ": 14, "ओ": 15, "औ": 16, "क": 17, "ख": 18, "ग": 19, "घ": 20, "च": 21, "छ": 22, "ज": 23, "झ": 24, "ञ": 25, "ट": 26, "ठ": 27, "ड": 28, "ढ": 29, "ण": 30, "त": 31, "थ": 32, "द": 33, "ध": 34, "न": 35, "प": 36, "फ": 37, "ब": 38, "भ": 39, "म": 40, "य": 41, "र": 42, "ल": 43, "व": 44, "श": 45, "ष": 46, "स": 47, "ह": 48, "़": 49, "ा": 50, "ि": 51, "ी": 52, "ु": 53, "ू": 54, "ृ": 55, "े": 56, "ै": 57, "ॉ": 58, "ो": 59, "ौ": 60, "्": 61, "क़": 62, "ख़": 63, "ग़": 64, "ज़": 65, "ड़": 66, "ढ़": 67, "फ़": 68, "ॠ": 69, "|": 0, "[UNK]": 70, "[PAD]": 71}