DrishtiSharma's picture
add tokenizer
639a44a
raw
history blame contribute delete
793 Bytes
{"q": 1, "ँ": 2, "ं": 3, "ः": 4, "अ": 5, "आ": 6, "इ": 7, "ई": 8, "उ": 9, "ऊ": 10, "ऋ": 11, "ए": 12, "ऐ": 13, "ऑ": 14, "ओ": 15, "औ": 16, "क": 17, "ख": 18, "ग": 19, "घ": 20, "च": 21, "छ": 22, "ज": 23, "झ": 24, "ञ": 25, "ट": 26, "ठ": 27, "ड": 28, "ढ": 29, "ण": 30, "त": 31, "थ": 32, "द": 33, "ध": 34, "न": 35, "प": 36, "फ": 37, "ब": 38, "भ": 39, "म": 40, "य": 41, "र": 42, "ल": 43, "व": 44, "श": 45, "ष": 46, "स": 47, "ह": 48, "़": 49, "ा": 50, "ि": 51, "ी": 52, "ु": 53, "ू": 54, "ृ": 55, "ॅ": 56, "े": 57, "ै": 58, "ॉ": 59, "ो": 60, "ौ": 61, "्": 62, "क़": 63, "ख़": 64, "ग़": 65, "ज़": 66, "ड़": 67, "ढ़": 68, "फ़": 69, "–": 70, "|": 0, "[UNK]": 71, "[PAD]": 72}