whispherMusic / added_tokens.json
nacielo's picture
Upload tokenizer
f053505
{
"<|af|>": 50326,
"<|am|>": 50333,
"<|ar|>": 50271,
"<|as|>": 50349,
"<|az|>": 50303,
"<|ba|>": 50354,
"<|be|>": 50329,
"<|bg|>": 50291,
"<|bn|>": 50301,
"<|bo|>": 50346,
"<|br|>": 50308,
"<|bs|>": 50314,
"<|ca|>": 50269,
"<|cs|>": 50282,
"<|cy|>": 50296,
"<|da|>": 50284,
"<|de|>": 50260,
"<|el|>": 50280,
"<|en|>": 50258,
"<|es|>": 50261,
"<|et|>": 50306,
"<|eu|>": 50309,
"<|fa|>": 50299,
"<|fi|>": 50276,
"<|fo|>": 50337,
"<|fr|>": 50264,
"<|gl|>": 50318,
"<|gu|>": 50332,
"<|haw|>": 50351,
"<|ha|>": 50353,
"<|hi|>": 50275,
"<|hr|>": 50290,
"<|ht|>": 50338,
"<|hu|>": 50285,
"<|hy|>": 50311,
"<|id|>": 50274,
"<|is|>": 50310,
"<|it|>": 50273,
"<|iw|>": 50278,
"<|ja|>": 50265,
"<|jw|>": 50355,
"<|ka|>": 50328,
"<|kk|>": 50315,
"<|km|>": 50322,
"<|kn|>": 50305,
"<|ko|>": 50263,
"<|la|>": 50293,
"<|lb|>": 50344,
"<|ln|>": 50352,
"<|lo|>": 50335,
"<|lt|>": 50292,
"<|lv|>": 50300,
"<|mg|>": 50348,
"<|mi|>": 50294,
"<|mk|>": 50307,
"<|ml|>": 50295,
"<|mn|>": 50313,
"<|mr|>": 50319,
"<|ms|>": 50281,
"<|mt|>": 50342,
"<|my|>": 50345,
"<|ne|>": 50312,
"<|nl|>": 50270,
"<|nn|>": 50341,
"<|nocaptions|>": 50361,
"<|notimestamps|>": 50362,
"<|no|>": 50287,
"<|oc|>": 50327,
"<|pa|>": 50320,
"<|pl|>": 50268,
"<|ps|>": 50339,
"<|pt|>": 50266,
"<|ro|>": 50283,
"<|ru|>": 50262,
"<|sa|>": 50343,
"<|sd|>": 50331,
"<|si|>": 50321,
"<|sk|>": 50297,
"<|sl|>": 50304,
"<|sn|>": 50323,
"<|so|>": 50325,
"<|sq|>": 50316,
"<|sr|>": 50302,
"<|startoflm|>": 50359,
"<|startofprev|>": 50360,
"<|startoftranscript|>": 50257,
"<|su|>": 50356,
"<|sv|>": 50272,
"<|sw|>": 50317,
"<|ta|>": 50286,
"<|te|>": 50298,
"<|tg|>": 50330,
"<|th|>": 50288,
"<|tk|>": 50340,
"<|tl|>": 50347,
"<|transcribe|>": 50358,
"<|translate|>": 50357,
"<|tr|>": 50267,
"<|tt|>": 50350,
"<|uk|>": 50279,
"<|ur|>": 50289,
"<|uz|>": 50336,
"<|vi|>": 50277,
"<|yi|>": 50334,
"<|yo|>": 50324,
"<|zh|>": 50259
}