Upload tokenizer

special_tokens_map.json

@@ -1,6 +1,6 @@
 {
   "bos_token": "<s>",
   "eos_token": "</s>",
-  "pad_token": "[PAD]",
-  "unk_token": "[UNK]"
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
 }

tokenizer_config.json

@@ -1,31 +1,31 @@
 {
   "added_tokens_decoder": {
-    "55": {
-      "content": "[UNK]",
+    "0": {
+      "content": "<pad>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "56": {
-      "content": "[PAD]",
+    "1": {
+      "content": "<s>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "57": {
-      "content": "<s>",
+    "2": {
+      "content": "</s>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "58": {
-      "content": "</s>",
+    "3": {
+      "content": "<unk>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -35,14 +35,14 @@
   },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": true,
-  "do_phonemize": true,
+  "do_phonemize": false,
   "eos_token": "</s>",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "[PAD]",
+  "pad_token": "<pad>",
   "phone_delimiter_token": " ",
   "phonemizer_backend": "espeak",
   "phonemizer_lang": "en-us",
   "tokenizer_class": "Wav2Vec2PhonemeCTCTokenizer",
-  "unk_token": "[UNK]",
-  "word_delimiter_token": "|"
+  "unk_token": "<unk>",
+  "word_delimiter_token": null
 }

vocab.json

@@ -1,59 +1,394 @@
 {
-  ".": 31,
-  "1": 24,
-  "2": 2,
-  "5": 44,
-  "[PAD]": 56,
-  "[UNK]": 55,
-  "a": 11,
-  "b": 7,
-  "d": 47,
-  "e": 33,
-  "f": 28,
-  "h": 37,
-  "i": 45,
-  "j": 23,
-  "k": 12,
+  "1": 184,
+  "</s>": 2,
+  "<pad>": 0,
+  "<s>": 1,
+  "<unk>": 3,
+  "??": 85,
+  "N": 280,
+  "S": 208,
+  "X": 326,
+  "a": 9,
+  "a.": 250,
+  "a.ː": 251,
+  "a1": 354,
+  "a2": 187,
+  "a4": 391,
+  "a5": 106,
+  "ai": 265,
+  "ai2": 231,
+  "ai5": 130,
+  "aiɜ": 181,
+  "au": 282,
+  "aɜ": 160,
+  "aɨ": 193,
+  "aɪ": 37,
+  "aɪə": 139,
+  "aɪɚ": 131,
+  "aʊ": 53,
+  "aː": 73,
+  "ã": 330,
+  "b": 26,
+  "bʰ": 296,
+  "bʲ": 178,
+  "bː": 239,
+  "c": 200,
+  "cʰ": 356,
+  "cʰcʰ": 371,
+  "cː": 334,
+  "d": 12,
+  "dZ": 216,
+  "d[": 152,
+  "d^": 300,
+  "dzː": 382,
+  "dʑ": 121,
+  "dʑʲ": 345,
+  "dʒ": 60,
+  "dʒʲ": 324,
+  "dʒː": 338,
+  "dʰ": 303,
+  "dʰː": 384,
+  "dʲ": 115,
+  "dʲʲ": 383,
+  "dː": 253,
+  "dˤ": 205,
+  "dˤdˤ": 337,
+  "e": 16,
+  "e:": 319,
+  "ee": 255,
+  "ei2": 198,
+  "ei5": 125,
+  "eiɜ": 212,
+  "eɑ": 317,
+  "eə": 269,
+  "eɪ": 44,
+  "eʊ": 163,
+  "eː": 51,
+  "ẽ": 360,
+  "ẽː": 332,
+  "e̞": 123,
+  "e̞e̞": 380,
+  "f": 23,
+  "fʲ": 230,
+  "h": 39,
+  "i": 10,
+  "i.": 270,
+  "i.1": 335,
+  "i.2": 218,
+  "i.4": 388,
+  "i.5": 99,
+  "i.ɜ": 162,
+  "i.ː": 261,
+  "i1": 306,
+  "i2": 135,
+  "i4": 339,
+  "i5": 82,
+  "i:": 389,
+  "ie": 272,
+  "iou1": 352,
+  "iou2": 199,
+  "iou4": 355,
+  "iou5": 215,
+  "iouɜ": 190,
+  "iɑ1": 366,
+  "iɑ2": 227,
+  "iɑ5": 129,
+  "iɑɜ": 223,
+  "iə": 81,
+  "iɛ1": 357,
+  "iɛ2": 194,
+  "iɛ4": 379,
+  "iɛ5": 107,
+  "iɛɜ": 146,
+  "iɜ": 105,
+  "iʊ": 290,
+  "iː": 30,
+  "iː1": 373,
+  "iːː": 364,
+  "ĩ": 314,
+  "i̪1": 281,
+  "i̪2": 247,
+  "i̪4": 310,
+  "i̪5": 157,
+  "i̪ɜ": 258,
+  "j": 24,
+  "ja": 110,
+  "ju": 141,
+  "k": 11,
+  "kh": 132,
+  "kʰ": 288,
+  "kʰː": 375,
+  "kʲ": 168,
+  "kː": 201,
   "l": 8,
-  "m": 34,
-  "n": 16,
-  "o": 54,
-  "p": 14,
-  "r": 46,
-  "s": 43,
-  "t": 5,
-  "u": 48,
-  "v": 20,
-  "w": 22,
-  "x": 4,
-  "y": 32,
-  "z": 18,
-  "|": 19,
-  "æ": 3,
-  "ð": 35,
-  "ŋ": 26,
-  "ɐ": 51,
-  "ɑ": 25,
-  "ɔ": 10,
-  "ɕ": 40,
-  "ə": 15,
-  "ɚ": 52,
-  "ɛ": 39,
-  "ɜ": 30,
-  "ɡ": 38,
-  "ɪ": 13,
-  "ɹ": 29,
-  "ɾ": 50,
-  "ʃ": 21,
-  "ʊ": 42,
-  "ʌ": 53,
-  "ʐ": 6,
-  "ʒ": 1,
-  "ʔ": 41,
-  "ʲ": 36,
-  "ː": 17,
-  "̩": 0,
-  "̪": 49,
-  "θ": 9,
-  "ᵻ": 27
+  "lː": 386,
+  "l̩": 243,
+  "m": 13,
+  "mʲ": 119,
+  "n": 4,
+  "nʲ": 86,
+  "nʲʲ": 309,
+  "n̩": 169,
+  "o": 19,
+  "o1": 263,
+  "o2": 164,
+  "o4": 275,
+  "o5": 126,
+  "o:": 277,
+  "oe": 204,
+  "oe:": 305,
+  "onɡ2": 210,
+  "onɡ5": 127,
+  "onɡɜ": 192,
+  "ou1": 378,
+  "ou2": 228,
+  "ou5": 147,
+  "ouɜ": 248,
+  "oɜ": 144,
+  "oɪ": 114,
+  "oʊ": 49,
+  "oː": 55,
+  "oːɹ": 77,
+  "õ": 292,
+  "o̞": 197,
+  "o̞o̞": 376,
+  "p": 18,
+  "pf": 225,
+  "ph": 171,
+  "pʰ": 344,
+  "pʲ": 167,
+  "pː": 145,
+  "q": 101,
+  "qː": 285,
+  "r": 31,
+  "r.": 304,
+  "rʲ": 96,
+  "r̝": 232,
+  "r̝̊": 217,
+  "r̩": 254,
+  "s": 5,
+  "s.": 83,
+  "s^": 220,
+  "sx": 291,
+  "sʲ": 102,
+  "s̪": 180,
+  "s̪ː": 301,
+  "t": 6,
+  "tS": 219,
+  "t[": 138,
+  "t^": 262,
+  "t^ː": 331,
+  "th": 112,
+  "ts": 54,
+  "ts.": 95,
+  "ts.h": 118,
+  "tsh": 154,
+  "tsʲ": 312,
+  "tsː": 320,
+  "tɕ": 84,
+  "tɕh": 117,
+  "tɕʲ": 329,
+  "tʃ": 66,
+  "tʃʰ": 289,
+  "tʃʲ": 124,
+  "tʃː": 276,
+  "tʰ": 294,
+  "tʲ": 92,
+  "tː": 150,
+  "t̪": 189,
+  "t̪ː": 316,
+  "u": 34,
+  "u\"": 211,
+  "u.": 284,
+  "u.ː": 297,
+  "u1": 279,
+  "u2": 143,
+  "u4": 318,
+  "u5": 103,
+  "u:": 260,
+  "ua1": 347,
+  "ua2": 273,
+  "ua4": 348,
+  "ua5": 236,
+  "uai5": 295,
+  "uaiɜ": 328,
+  "uaɜ": 249,
+  "uei2": 311,
+  "uei5": 158,
+  "ueiɜ": 266,
+  "ui": 351,
+  "uo": 293,
+  "uo1": 387,
+  "uo2": 241,
+  "uo5": 151,
+  "uoɜ": 153,
+  "uə2": 298,
+  "uə5": 224,
+  "uəɜ": 233,
+  "uɜ": 149,
+  "uɨ": 173,
+  "uɪ": 237,
+  "uː": 46,
+  "ũ": 191,
+  "v": 25,
+  "vʲ": 122,
+  "w": 32,
+  "x": 62,
+  "xʲ": 377,
+  "y": 47,
+  "y1": 381,
+  "y2": 202,
+  "y5": 148,
+  "y:": 287,
+  "yi": 341,
+  "yiɜ": 302,
+  "yu2": 308,
+  "yu5": 240,
+  "yuɜ": 346,
+  "yæ2": 283,
+  "yæ5": 252,
+  "yæɜ": 186,
+  "yə2": 358,
+  "yə5": 222,
+  "yəɜ": 286,
+  "yɛ2": 333,
+  "yɛ5": 242,
+  "yɛ5ʲ": 390,
+  "yɛɜ": 206,
+  "yɜ": 177,
+  "yː": 79,
+  "z": 21,
+  "ä": 166,
+  "ää": 362,
+  "æ": 36,
+  "æi": 259,
+  "æiː": 349,
+  "æː": 267,
+  "ç": 64,
+  "ð": 22,
+  "ø": 76,
+  "øi": 369,
+  "øː": 108,
+  "ħ": 137,
+  "ŋ": 42,
+  "œ": 75,
+  "œː": 368,
+  "œ̃": 88,
+  "ũ": 374,
+  "ɐ": 20,
+  "ɐɐ": 350,
+  "ɐ̃": 165,
+  "ɐ̃ʊ̃": 183,
+  "ɑ": 59,
+  "ɑ1": 274,
+  "ɑ2": 179,
+  "ɑ4": 315,
+  "ɑ5": 113,
+  "ɑ:": 278,
+  "ɑu2": 172,
+  "ɑu5": 128,
+  "ɑuɜ": 196,
+  "ɑɜ": 155,
+  "ɑɨ": 161,
+  "ɑː": 41,
+  "ɑːɹ": 68,
+  "ɑ̃": 48,
+  "ɒ": 170,
+  "ɔ": 40,
+  "ɔø": 90,
+  "ɔɨ": 195,
+  "ɔɪ": 100,
+  "ɔː": 67,
+  "ɔːɹ": 71,
+  "ɔ̃": 56,
+  "ɕ": 74,
+  "ɕʲ": 342,
+  "ɖ": 203,
+  "ɖʰ": 359,
+  "ə": 7,
+  "ə1": 133,
+  "ə2": 142,
+  "ə4": 185,
+  "ə5": 116,
+  "əl": 61,
+  "ər1": 367,
+  "ər2": 229,
+  "ər4": 363,
+  "ər5": 257,
+  "ərɜ": 245,
+  "əɜ": 111,
+  "əɨ": 235,
+  "əɪ": 174,
+  "əʊ": 207,
+  "əː1": 372,
+  "ɚ": 43,
+  "ɛ": 14,
+  "ɛɪ": 268,
+  "ɛɹ": 94,
+  "ɛʊ": 336,
+  "ɛː": 87,
+  "ɛ̃": 69,
+  "ɜ": 58,
+  "ɜː": 63,
+  "ɟ": 159,
+  "ɟʰ": 343,
+  "ɟː": 307,
+  "ɡ": 35,
+  "ɡʰ": 322,
+  "ɡʲ": 209,
+  "ɡː": 340,
+  "ɣ": 57,
+  "ɨ": 93,
+  "ɨu": 226,
+  "ɨː": 188,
+  "ɪ": 17,
+  "ɪ^": 264,
+  "ɪu": 256,
+  "ɪuː": 370,
+  "ɪɹ": 97,
+  "ɪː": 365,
+  "ɫ": 175,
+  "ɬ": 176,
+  "ɭ": 89,
+  "ɭʲ": 98,
+  "ɯ": 120,
+  "ɯɯ": 323,
+  "ɯᵝ": 214,
+  "ɯᵝɯᵝ": 385,
+  "ɲ": 78,
+  "ɳ": 234,
+  "ɴ": 271,
+  "ɵ": 156,
+  "ɵː": 327,
+  "ɸ": 313,
+  "ɹ": 27,
+  "ɻ": 244,
+  "ɽ": 238,
+  "ɾ": 15,
+  "ʁ": 28,
+  "ʂ": 246,
+  "ʂʲ": 325,
+  "ʃ": 38,
+  "ʈ": 213,
+  "ʈʰ": 321,
+  "ʉ": 136,
+  "ʊ": 29,
+  "ʊə": 299,
+  "ʊɹ": 104,
+  "ʊː": 353,
+  "ʋ": 72,
+  "ʌ": 33,
+  "ʎ": 70,
+  "ʐ": 134,
+  "ʑ": 91,
+  "ʒ": 65,
+  "ʒʲ": 361,
+  "ʔ": 80,
+  "ʕ": 109,
+  "ʝ": 221,
+  "ʲ": 140,
+  "β": 45,
+  "θ": 52,
+  "χ": 182,
+  "ᵻ": 50
 }