Upload tokenizer_config.json

tokenizer_config.json

@@ -1,119 +1 @@
-{
-  "additional_special_tokens": [
-    "__af__",
-    "__am__",
-    "__ar__",
-    "__ast__",
-    "__az__",
-    "__ba__",
-    "__be__",
-    "__bg__",
-    "__bn__",
-    "__br__",
-    "__bs__",
-    "__ca__",
-    "__ceb__",
-    "__cs__",
-    "__cy__",
-    "__da__",
-    "__de__",
-    "__el__",
-    "__en__",
-    "__es__",
-    "__et__",
-    "__fa__",
-    "__ff__",
-    "__fi__",
-    "__fr__",
-    "__fy__",
-    "__ga__",
-    "__gd__",
-    "__gl__",
-    "__gu__",
-    "__ha__",
-    "__he__",
-    "__hi__",
-    "__hr__",
-    "__ht__",
-    "__hu__",
-    "__hy__",
-    "__id__",
-    "__ig__",
-    "__ilo__",
-    "__is__",
-    "__it__",
-    "__ja__",
-    "__jv__",
-    "__ka__",
-    "__kk__",
-    "__km__",
-    "__kn__",
-    "__ko__",
-    "__lb__",
-    "__lg__",
-    "__ln__",
-    "__lo__",
-    "__lt__",
-    "__lv__",
-    "__mg__",
-    "__mk__",
-    "__ml__",
-    "__mn__",
-    "__mr__",
-    "__ms__",
-    "__my__",
-    "__ne__",
-    "__nl__",
-    "__no__",
-    "__ns__",
-    "__oc__",
-    "__or__",
-    "__pa__",
-    "__pl__",
-    "__ps__",
-    "__pt__",
-    "__ro__",
-    "__ru__",
-    "__sd__",
-    "__si__",
-    "__sk__",
-    "__sl__",
-    "__so__",
-    "__sq__",
-    "__sr__",
-    "__ss__",
-    "__su__",
-    "__sv__",
-    "__sw__",
-    "__ta__",
-    "__th__",
-    "__tl__",
-    "__tn__",
-    "__tr__",
-    "__uk__",
-    "__ur__",
-    "__uz__",
-    "__vi__",
-    "__wo__",
-    "__xh__",
-    "__yi__",
-    "__yo__",
-    "__zh__",
-    "__zu__"
-  ],
-  "bos_token": "<s>",
-  "eos_token": "</s>",
-  "language_codes": "m2m100",
-  "model_max_length": 1024,
-  "name_or_path": "facebook/m2m100_418M",
-  "num_madeup_words": 8,
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
-  "sp_model_kwargs": {},
-  "special_tokens_map_file": "m2m_100_1.2B_v2/special_tokens_map.json",
-  "src_lang": null,
-  "tgt_lang": null,
-  "tokenizer_class": "M2M100Tokenizer",
-  "tokenizer_file": null,
-  "unk_token": "<unk>"
-}
+{"src_lang": null, "tgt_lang": null, "bos_token": "<s>", "eos_token": "</s>", "sep_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "special_tokens_map_file": "m2m_100_1.2B_v2/special_tokens_map.json", "tokenizer_file": null, "name_or_path": "m2m_100_1.2B_v2"}