codingutf8 2019 the open ai team s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for xlm import json import os import re import sys import unicodedata from typing import list optional tuple from tokenizationutils import pretrainedtokenizer from utils import logging logger logging getloggername vocabfilesnames vocabfile vocab json mergesfile merges txt pretrainedvocabfilesmap vocabfile xlmmlmen2048 https huggingface coxlmmlmen2048resolvemainvocab json xlmmlmende1024 https huggingface coxlmmlmende1024resolvemainvocab json xlmmlmenfr1024 https huggingface coxlmmlmenfr1024resolvemainvocab json xlmmlmenro1024 https huggingface coxlmmlmenro1024resolvemainvocab json xlmmlmtlmxnli151024 https huggingface coxlmmlmtlmxnli151024resolvemainvocab json xlmmlmxnli151024 https huggingface coxlmmlmxnli151024resolvemainvocab json xlmclmenfr1024 https huggingface coxlmclmenfr1024resolvemainvocab json xlmclmende1024 https huggingface coxlmclmende1024resolvemainvocab json xlmmlm171280 https huggingface coxlmmlm171280resolvemainvocab json xlmmlm1001280 https huggingface coxlmmlm1001280resolvemainvocab json mergesfile xlmmlmen2048 https huggingface coxlmmlmen2048resolvemainmerges txt xlmmlmende1024 https huggingface coxlmmlmende1024resolvemainmerges txt xlmmlmenfr1024 https huggingface coxlmmlmenfr1024resolvemainmerges txt xlmmlmenro1024 https huggingface coxlmmlmenro1024resolvemainmerges txt xlmmlmtlmxnli151024 https huggingface coxlmmlmtlmxnli151024resolvemainmerges txt xlmmlmxnli151024 https huggingface coxlmmlmxnli151024resolvemainmerges txt xlmclmenfr1024 https huggingface coxlmclmenfr1024resolvemainmerges txt xlmclmende1024 https huggingface coxlmclmende1024resolvemainmerges txt xlmmlm171280 https huggingface coxlmmlm171280resolvemainmerges txt xlmmlm1001280 https huggingface coxlmmlm1001280resolvemainmerges txt pretrainedpositionalembeddingssizes xlmmlmen2048 512 xlmmlmende1024 512 xlmmlmenfr1024 512 xlmmlmenro1024 512 xlmmlmtlmxnli151024 512 xlmmlmxnli151024 512 xlmclmenfr1024 512 xlmclmende1024 512 xlmmlm171280 512 xlmmlm1001280 512 pretrainedinitconfiguration xlmmlmen2048 dolowercaseandremoveaccent true xlmmlmende1024 dolowercaseandremoveaccent true id2lang 0 de 1 en lang2id de 0 en 1 xlmmlmenfr1024 dolowercaseandremoveaccent true id2lang 0 en 1 fr lang2id en 0 fr 1 xlmmlmenro1024 dolowercaseandremoveaccent true id2lang 0 en 1 ro lang2id en 0 ro 1 xlmmlmtlmxnli151024 dolowercaseandremoveaccent true id2lang 0 ar 1 bg 2 de 3 el 4 en 5 es 6 fr 7 hi 8 ru 9 sw 10 th 11 tr 12 ur 13 vi 14 zh lang2id ar 0 bg 1 de 2 el 3 en 4 es 5 fr 6 hi 7 ru 8 sw 9 th 10 tr 11 ur 12 vi 13 zh 14 xlmmlmxnli151024 dolowercaseandremoveaccent true id2lang 0 ar 1 bg 2 de 3 el 4 en 5 es 6 fr 7 hi 8 ru 9 sw 10 th 11 tr 12 ur 13 vi 14 zh lang2id ar 0 bg 1 de 2 el 3 en 4 es 5 fr 6 hi 7 ru 8 sw 9 th 10 tr 11 ur 12 vi 13 zh 14 xlmclmenfr1024 dolowercaseandremoveaccent true id2lang 0 en 1 fr lang2id en 0 fr 1 xlmclmende1024 dolowercaseandremoveaccent true id2lang 0 de 1 en lang2id de 0 en 1 xlmmlm171280 dolowercaseandremoveaccent false id2lang 0 ar 1 de 2 en 3 es 4 fr 5 hi 6 it 7 ja 8 ko 9 nl 10 pl 11 pt 12 ru 13 sv 14 tr 15 vi 16 zh lang2id ar 0 de 1 en 2 es 3 fr 4 hi 5 it 6 ja 7 ko 8 nl 9 pl 10 pt 11 ru 12 sv 13 tr 14 vi 15 zh 16 xlmmlm1001280 dolowercaseandremoveaccent false id2lang 0 af 1 als 2 am 3 an 4 ang 5 ar 6 arz 7 ast 8 az 9 bar 10 be 11 bg 12 bn 13 br 14 bs 15 ca 16 ceb 17 ckb 18 cs 19 cy 20 da 21 de 22 el 23 en 24 eo 25 es 26 et 27 eu 28 fa 29 fi 30 fr 31 fy 32 ga 33 gan 34 gl 35 gu 36 he 37 hi 38 hr 39 hu 40 hy 41 ia 42 id 43 is 44 it 45 ja 46 jv 47 ka 48 kk 49 kn 50 ko 51 ku 52 la 53 lb 54 lt 55 lv 56 mk 57 ml 58 mn 59 mr 60 ms 61 my 62 nds 63 ne 64 nl 65 nn 66 no 67 oc 68 pl 69 pt 70 ro 71 ru 72 scn 73 sco 74 sh 75 si 76 simple 77 sk 78 sl 79 sq 80 sr 81 sv 82 sw 83 ta 84 te 85 th 86 tl 87 tr 88 tt 89 uk 90 ur 91 uz 92 vi 93 war 94 wuu 95 yi 96 zh 97 zhclassical 98 zhminnan 99 zhyue lang2id af 0 als 1 am 2 an 3 ang 4 ar 5 arz 6 ast 7 az 8 bar 9 be 10 bg 11 bn 12 br 13 bs 14 ca 15 ceb 16 ckb 17 cs 18 cy 19 da 20 de 21 el 22 en 23 eo 24 es 25 et 26 eu 27 fa 28 fi 29 fr 30 fy 31 ga 32 gan 33 gl 34 gu 35 he 36 hi 37 hr 38 hu 39 hy 40 ia 41 id 42 is 43 it 44 ja 45 jv 46 ka 47 kk 48 kn 49 ko 50 ku 51 la 52 lb 53 lt 54 lv 55 mk 56 ml 57 mn 58 mr 59 ms 60 my 61 nds 62 ne 63 nl 64 nn 65 no 66 oc 67 pl 68 pt 69 ro 70 ru 71 scn 72 sco 73 sh 74 si 75 simple 76 sk 77 sl 78 sq 79 sr 80 sv 81 sw 82 ta 83 te 84 th 85 tl 86 tr 87 tt 88 uk 89 ur 90 uz 91 vi 92 war 93 wuu 94 yi 95 zh 96 zhclassical 97 zhminnan 98 zhyue 99 def getpairsword pairs set prevchar word0 for char in word1 pairs addprevchar char prevchar char return pairs def lowercaseandremoveaccenttext text jointext text text lower text unicodedata normalizenfd text output for char in text cat unicodedata categorychar if cat mn continue output appendchar return joinoutput lower split def replaceunicodepuncttext text text replace text re subrs text text text replace text text replace text text replace text text replace text text replace text text replace text text replace text text replace text text replace text text replace text text replace text text replace text text replace 1 text text replace text text replace text text replace 0 text text replace 3 text text replace 2 text text replace 5 text text replace 6 text text replace 9 text text replace 7 text text replace 8 text text replace 4 text re subrs text text text replace text text replace text text replace text text replace text text replace text text replace text text replace text text replace text text replace return text def removenonprintingchartext output for char in text cat unicodedata categorychar if cat startswithc continue output appendchar return joinoutput def romanianpreprocessingtext https github comrsennrichwmt16scriptsblobmasterpreprocessnormaliseromanian py https github comrsennrichwmt16scriptsblobmasterpreprocessremovediacritics py construct an xlm tokenizer based on bytepair encoding the tokenization process is the following moses preprocessing and tokenization for most supported languages language specific tokenization for chinese jieba japanese kytea and thai pythainlp optionally lowercases and normalizes all inputs text the arguments specialtokens and the function setspecialtokens can be used to add additional symbols like classify to a vocabulary the lang2id attribute maps the languages supported by the model with their ids if provided automatically set for pretrained vocabularies the id2lang attributes does reverse mapping if provided automatically set for pretrained vocabularies this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocabfile str vocabulary file mergesfile str merges file unktoken str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead bostoken str optional defaults to s the beginning of sequence token that was used during pretraining can be used a sequence classifier token tip when building a sequence using special tokens this is not the token that is used for the beginning of sequence the token used is the clstoken tip septoken str optional defaults to s the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens padtoken str optional defaults to pad the token used for padding for example when batching sequences of different lengths clstoken str optional defaults to s the classifier token which is used when doing sequence classification classification of the whole sequence instead of pertoken classification it is the first token of the sequence when built with special tokens masktoken str optional defaults to special1 the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict additionalspecialtokens liststr optional defaults to special0 special1 special2 special3 special4 special5 special6 special7 special8 special9 list of additional special tokens lang2id dictstr int optional dictionary mapping languages string identifiers to their ids id2lang dictint str optional dictionary mapping language ids to their string identifiers dolowercaseandremoveaccent bool optional defaults to true whether to lowercase and remove accents when tokenizing cache of sm mosespunctnormalizer instance cache of sm mosestokenizer instance true for current supported model v1 2 0 false for xlm17 100 tokenize a string given language code for chinese japanese and thai we use a language specific tokenizer otherwise we use moses details of tokenization sacremoseshttps github comalvationssacremoses port of moses install with pip install sacremoses pythainlphttps github compythainlppythainlp thai tokenizer install with pip install pythainlp kyteahttps github comchezoumykyteapython japanese tokenizer wrapper of kyteahttps github comneubigkytea install with the following steps git clone gitgithub com neubigkytea git cd kytea autoreconf i configure prefixhomelocal make make install pip install kytea jiebahttps github comfxsjyjieba chinese tokenizer install with pip install jieba the original xlm used stanford segmenterhttps nlp stanford edusoftwarestanfordsegmenter20181016 zip however the wrapper nltk tokenize stanfordsegmenter is slow due to jvm overhead and it will be deprecated jieba is a lot faster and pipinstallable note there is some mismatch with the stanford segmenter it should be fine if you finetune the model with chinese supervisionself if you want the same exact behaviour use the original xlm preprocessing scripthttps github comfacebookresearchxlmtreemastertools to tokenize the sentence externally and set bypasstokenizertrue to bypass the tokenizer args lang iso language code default en string languages should belong of the model supported languages however we don t enforce it bypasstokenizer allow users to preprocess and tokenize the sentences externally default false bool if true we only apply bpe returns list of tokens todo make sure we are using xlmmlmenro1024 since xlm100 doesn t have this step converts a token str in an id using the vocab return self encoder gettoken self encoder getself unktoken def convertidtotokenself index converts a sequence of tokens string in a single string outstring jointokens replacew strip return outstring def buildinputswithspecialtokens self tokenids0 listint tokenids1 optionallistint none listint bos self bostokenid sep self septokenid if tokenids1 is none return bos tokenids0 sep return bos tokenids0 sep tokenids1 sep def getspecialtokensmask self tokenids0 listint tokenids1 optionallistint none alreadyhasspecialtokens bool false listint if alreadyhasspecialtokens return super getspecialtokensmask tokenids0tokenids0 tokenids1tokenids1 alreadyhasspecialtokenstrue if tokenids1 is not none return 1 0 lentokenids0 1 0 lentokenids1 1 return 1 0 lentokenids0 1 def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint sep self septokenid cls self clstokenid if tokenids1 is none return lencls tokenids0 sep 0 return lencls tokenids0 sep 0 lentokenids1 sep 1 def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr if not os path isdirsavedirectory logger errorfvocabulary path savedirectory should be a directory return vocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesvocabfile mergefile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesmergesfile with openvocabfile w encodingutf8 as f f writejson dumpsself encoder indent2 sortkeystrue ensureasciifalse n index 0 with openmergefile w encodingutf8 as writer for bpetokens tokenindex in sortedself bperanks items keylambda kv kv1 if index tokenindex logger warning fsaving vocabulary to mergefile bpe merge indices are not consecutive please check that the tokenizer is not corrupted index tokenindex writer write joinbpetokens n index 1 return vocabfile mergefile def getstateself state self dict copy statesm none return state def setstateself d self dict d try import sacremoses except importerror raise importerror you need to install sacremoses to use xlmtokenizer see https pypi orgprojectsacremoses for installation self sm sacremoses coding utf 8 2019 the open ai team s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for xlm return set of symbol pairs in a word word is represented as tuple of symbols symbols being variable length strings lowercase and strips accents from a piece of text based on https github com facebookresearch xlm blob master tools lowercase_and_remove_accent py port of https github com moses smt mosesdecoder blob master scripts tokenizer replace unicode punctuation perl port of https github com moses smt mosesdecoder blob master scripts tokenizer remove non printing char perl sennrich s wmt16 scripts for romanian preprocessing used by model xlm mlm enro 1024 https github com rsennrich wmt16 scripts blob master preprocess normalise romanian py https github com rsennrich wmt16 scripts blob master preprocess remove diacritics py s comma t comma construct an xlm tokenizer based on byte pair encoding the tokenization process is the following moses preprocessing and tokenization for most supported languages language specific tokenization for chinese jieba japanese kytea and thai pythainlp optionally lowercases and normalizes all inputs text the arguments special_tokens and the function set_special_tokens can be used to add additional symbols like __classify__ to a vocabulary the lang2id attribute maps the languages supported by the model with their ids if provided automatically set for pretrained vocabularies the id2lang attributes does reverse mapping if provided automatically set for pretrained vocabularies this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str vocabulary file merges_file str merges file unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead bos_token str optional defaults to s the beginning of sequence token that was used during pretraining can be used a sequence classifier token tip when building a sequence using special tokens this is not the token that is used for the beginning of sequence the token used is the cls_token tip sep_token str optional defaults to s the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths cls_token str optional defaults to s the classifier token which is used when doing sequence classification classification of the whole sequence instead of per token classification it is the first token of the sequence when built with special tokens mask_token str optional defaults to special1 the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict additional_special_tokens list str optional defaults to special0 special1 special2 special3 special4 special5 special6 special7 special8 special9 list of additional special tokens lang2id dict str int optional dictionary mapping languages string identifiers to their ids id2lang dict int str optional dictionary mapping language ids to their string identifiers do_lowercase_and_remove_accent bool optional defaults to true whether to lowercase and remove accents when tokenizing cache of sm mosespunctnormalizer instance cache of sm mosestokenizer instance true for current supported model v1 2 0 false for xlm 17 100 tokenize a string given language code for chinese japanese and thai we use a language specific tokenizer otherwise we use moses details of tokenization sacremoses https github com alvations sacremoses port of moses install with pip install sacremoses pythainlp https github com pythainlp pythainlp thai tokenizer install with pip install pythainlp kytea https github com chezou mykytea python japanese tokenizer wrapper of kytea https github com neubig kytea install with the following steps git clone git github com neubig kytea git cd kytea autoreconf i configure prefix home local make make install pip install kytea jieba https github com fxsjy jieba chinese tokenizer install with pip install jieba the original xlm used stanford segmenter https nlp stanford edu software stanford segmenter 2018 10 16 zip however the wrapper nltk tokenize stanford_segmenter is slow due to jvm overhead and it will be deprecated jieba is a lot faster and pip installable note there is some mismatch with the stanford segmenter it should be fine if you fine tune the model with chinese supervisionself if you want the same exact behaviour use the original xlm preprocessing script https github com facebookresearch xlm tree master tools to tokenize the sentence externally and set bypass_tokenizer true to bypass the tokenizer args lang iso language code default en string languages should belong of the model supported languages however we don t enforce it bypass_tokenizer allow users to preprocess and tokenize the sentences externally default false bool if true we only apply bpe returns list of tokens todo make sure we are using xlm mlm enro 1024 since xlm 100 doesn t have this step converts a token str in an id using the vocab converts an index integer in a token str using the vocab converts a sequence of tokens string in a single string build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens an xlm sequence has the following format single sequence s x s pair of sequences s a s b s args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepare_for_model method args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs already_has_special_tokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns list int a list of integers in the range 0 1 1 for a special token 0 for a sequence token create a mask from the two sequences passed to be used in a sequence pair classification task an xlm sequence pair mask has the following format 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 first sequence second sequence if token_ids_1 is none this method only returns the first portion of the mask 0s args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of token type ids glossary token type ids according to the given sequence s
import json
import os
import re
import sys
import unicodedata
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "xlm-mlm-en-2048": "https://huggingface.co/xlm-mlm-en-2048/resolve/main/vocab.json",
        "xlm-mlm-ende-1024": "https://huggingface.co/xlm-mlm-ende-1024/resolve/main/vocab.json",
        "xlm-mlm-enfr-1024": "https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/vocab.json",
        "xlm-mlm-enro-1024": "https://huggingface.co/xlm-mlm-enro-1024/resolve/main/vocab.json",
        "xlm-mlm-tlm-xnli15-1024": "https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/vocab.json",
        "xlm-mlm-xnli15-1024": "https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/vocab.json",
        "xlm-clm-enfr-1024": "https://huggingface.co/xlm-clm-enfr-1024/resolve/main/vocab.json",
        "xlm-clm-ende-1024": "https://huggingface.co/xlm-clm-ende-1024/resolve/main/vocab.json",
        "xlm-mlm-17-1280": "https://huggingface.co/xlm-mlm-17-1280/resolve/main/vocab.json",
        "xlm-mlm-100-1280": "https://huggingface.co/xlm-mlm-100-1280/resolve/main/vocab.json",
    },
    "merges_file": {
        "xlm-mlm-en-2048": "https://huggingface.co/xlm-mlm-en-2048/resolve/main/merges.txt",
        "xlm-mlm-ende-1024": "https://huggingface.co/xlm-mlm-ende-1024/resolve/main/merges.txt",
        "xlm-mlm-enfr-1024": "https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/merges.txt",
        "xlm-mlm-enro-1024": "https://huggingface.co/xlm-mlm-enro-1024/resolve/main/merges.txt",
        "xlm-mlm-tlm-xnli15-1024": "https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/merges.txt",
        "xlm-mlm-xnli15-1024": "https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/merges.txt",
        "xlm-clm-enfr-1024": "https://huggingface.co/xlm-clm-enfr-1024/resolve/main/merges.txt",
        "xlm-clm-ende-1024": "https://huggingface.co/xlm-clm-ende-1024/resolve/main/merges.txt",
        "xlm-mlm-17-1280": "https://huggingface.co/xlm-mlm-17-1280/resolve/main/merges.txt",
        "xlm-mlm-100-1280": "https://huggingface.co/xlm-mlm-100-1280/resolve/main/merges.txt",
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "xlm-mlm-en-2048": 512,
    "xlm-mlm-ende-1024": 512,
    "xlm-mlm-enfr-1024": 512,
    "xlm-mlm-enro-1024": 512,
    "xlm-mlm-tlm-xnli15-1024": 512,
    "xlm-mlm-xnli15-1024": 512,
    "xlm-clm-enfr-1024": 512,
    "xlm-clm-ende-1024": 512,
    "xlm-mlm-17-1280": 512,
    "xlm-mlm-100-1280": 512,
}

PRETRAINED_INIT_CONFIGURATION = {
    "xlm-mlm-en-2048": {"do_lowercase_and_remove_accent": True},
    "xlm-mlm-ende-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {0: "de", 1: "en"},
        "lang2id": {"de": 0, "en": 1},
    },
    "xlm-mlm-enfr-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {0: "en", 1: "fr"},
        "lang2id": {"en": 0, "fr": 1},
    },
    "xlm-mlm-enro-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {0: "en", 1: "ro"},
        "lang2id": {"en": 0, "ro": 1},
    },
    "xlm-mlm-tlm-xnli15-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {
            0: "ar",
            1: "bg",
            2: "de",
            3: "el",
            4: "en",
            5: "es",
            6: "fr",
            7: "hi",
            8: "ru",
            9: "sw",
            10: "th",
            11: "tr",
            12: "ur",
            13: "vi",
            14: "zh",
        },
        "lang2id": {
            "ar": 0,
            "bg": 1,
            "de": 2,
            "el": 3,
            "en": 4,
            "es": 5,
            "fr": 6,
            "hi": 7,
            "ru": 8,
            "sw": 9,
            "th": 10,
            "tr": 11,
            "ur": 12,
            "vi": 13,
            "zh": 14,
        },
    },
    "xlm-mlm-xnli15-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {
            0: "ar",
            1: "bg",
            2: "de",
            3: "el",
            4: "en",
            5: "es",
            6: "fr",
            7: "hi",
            8: "ru",
            9: "sw",
            10: "th",
            11: "tr",
            12: "ur",
            13: "vi",
            14: "zh",
        },
        "lang2id": {
            "ar": 0,
            "bg": 1,
            "de": 2,
            "el": 3,
            "en": 4,
            "es": 5,
            "fr": 6,
            "hi": 7,
            "ru": 8,
            "sw": 9,
            "th": 10,
            "tr": 11,
            "ur": 12,
            "vi": 13,
            "zh": 14,
        },
    },
    "xlm-clm-enfr-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {0: "en", 1: "fr"},
        "lang2id": {"en": 0, "fr": 1},
    },
    "xlm-clm-ende-1024": {
        "do_lowercase_and_remove_accent": True,
        "id2lang": {0: "de", 1: "en"},
        "lang2id": {"de": 0, "en": 1},
    },
    "xlm-mlm-17-1280": {
        "do_lowercase_and_remove_accent": False,
        "id2lang": {
            0: "ar",
            1: "de",
            2: "en",
            3: "es",
            4: "fr",
            5: "hi",
            6: "it",
            7: "ja",
            8: "ko",
            9: "nl",
            10: "pl",
            11: "pt",
            12: "ru",
            13: "sv",
            14: "tr",
            15: "vi",
            16: "zh",
        },
        "lang2id": {
            "ar": 0,
            "de": 1,
            "en": 2,
            "es": 3,
            "fr": 4,
            "hi": 5,
            "it": 6,
            "ja": 7,
            "ko": 8,
            "nl": 9,
            "pl": 10,
            "pt": 11,
            "ru": 12,
            "sv": 13,
            "tr": 14,
            "vi": 15,
            "zh": 16,
        },
    },
    "xlm-mlm-100-1280": {
        "do_lowercase_and_remove_accent": False,
        "id2lang": {
            0: "af",
            1: "als",
            2: "am",
            3: "an",
            4: "ang",
            5: "ar",
            6: "arz",
            7: "ast",
            8: "az",
            9: "bar",
            10: "be",
            11: "bg",
            12: "bn",
            13: "br",
            14: "bs",
            15: "ca",
            16: "ceb",
            17: "ckb",
            18: "cs",
            19: "cy",
            20: "da",
            21: "de",
            22: "el",
            23: "en",
            24: "eo",
            25: "es",
            26: "et",
            27: "eu",
            28: "fa",
            29: "fi",
            30: "fr",
            31: "fy",
            32: "ga",
            33: "gan",
            34: "gl",
            35: "gu",
            36: "he",
            37: "hi",
            38: "hr",
            39: "hu",
            40: "hy",
            41: "ia",
            42: "id",
            43: "is",
            44: "it",
            45: "ja",
            46: "jv",
            47: "ka",
            48: "kk",
            49: "kn",
            50: "ko",
            51: "ku",
            52: "la",
            53: "lb",
            54: "lt",
            55: "lv",
            56: "mk",
            57: "ml",
            58: "mn",
            59: "mr",
            60: "ms",
            61: "my",
            62: "nds",
            63: "ne",
            64: "nl",
            65: "nn",
            66: "no",
            67: "oc",
            68: "pl",
            69: "pt",
            70: "ro",
            71: "ru",
            72: "scn",
            73: "sco",
            74: "sh",
            75: "si",
            76: "simple",
            77: "sk",
            78: "sl",
            79: "sq",
            80: "sr",
            81: "sv",
            82: "sw",
            83: "ta",
            84: "te",
            85: "th",
            86: "tl",
            87: "tr",
            88: "tt",
            89: "uk",
            90: "ur",
            91: "uz",
            92: "vi",
            93: "war",
            94: "wuu",
            95: "yi",
            96: "zh",
            97: "zh_classical",
            98: "zh_min_nan",
            99: "zh_yue",
        },
        "lang2id": {
            "af": 0,
            "als": 1,
            "am": 2,
            "an": 3,
            "ang": 4,
            "ar": 5,
            "arz": 6,
            "ast": 7,
            "az": 8,
            "bar": 9,
            "be": 10,
            "bg": 11,
            "bn": 12,
            "br": 13,
            "bs": 14,
            "ca": 15,
            "ceb": 16,
            "ckb": 17,
            "cs": 18,
            "cy": 19,
            "da": 20,
            "de": 21,
            "el": 22,
            "en": 23,
            "eo": 24,
            "es": 25,
            "et": 26,
            "eu": 27,
            "fa": 28,
            "fi": 29,
            "fr": 30,
            "fy": 31,
            "ga": 32,
            "gan": 33,
            "gl": 34,
            "gu": 35,
            "he": 36,
            "hi": 37,
            "hr": 38,
            "hu": 39,
            "hy": 40,
            "ia": 41,
            "id": 42,
            "is": 43,
            "it": 44,
            "ja": 45,
            "jv": 46,
            "ka": 47,
            "kk": 48,
            "kn": 49,
            "ko": 50,
            "ku": 51,
            "la": 52,
            "lb": 53,
            "lt": 54,
            "lv": 55,
            "mk": 56,
            "ml": 57,
            "mn": 58,
            "mr": 59,
            "ms": 60,
            "my": 61,
            "nds": 62,
            "ne": 63,
            "nl": 64,
            "nn": 65,
            "no": 66,
            "oc": 67,
            "pl": 68,
            "pt": 69,
            "ro": 70,
            "ru": 71,
            "scn": 72,
            "sco": 73,
            "sh": 74,
            "si": 75,
            "simple": 76,
            "sk": 77,
            "sl": 78,
            "sq": 79,
            "sr": 80,
            "sv": 81,
            "sw": 82,
            "ta": 83,
            "te": 84,
            "th": 85,
            "tl": 86,
            "tr": 87,
            "tt": 88,
            "uk": 89,
            "ur": 90,
            "uz": 91,
            "vi": 92,
            "war": 93,
            "wuu": 94,
            "yi": 95,
            "zh": 96,
            "zh_classical": 97,
            "zh_min_nan": 98,
            "zh_yue": 99,
        },
    },
}


def get_pairs(word):
    
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs


def lowercase_and_remove_accent(text):
    
    text = " ".join(text)
    text = text.lower()
    text = unicodedata.normalize("NFD", text)
    output = []
    for char in text:
        cat = unicodedata.category(char)
        if cat == "Mn":
            continue
        output.append(char)
    return "".join(output).lower().split(" ")


def replace_unicode_punct(text):
    
    text = text.replace("，", ",")
    text = re.sub(r"。\s*", ". ", text)
    text = text.replace("、", ",")
    text = text.replace("”", '"')
    text = text.replace("“", '"')
    text = text.replace("∶", ":")
    text = text.replace("：", ":")
    text = text.replace("？", "?")
    text = text.replace("《", '"')
    text = text.replace("》", '"')
    text = text.replace("）", ")")
    text = text.replace("！", "!")
    text = text.replace("（", "(")
    text = text.replace("；", ";")
    text = text.replace("１", "1")
    text = text.replace("」", '"')
    text = text.replace("「", '"')
    text = text.replace("０", "0")
    text = text.replace("３", "3")
    text = text.replace("２", "2")
    text = text.replace("５", "5")
    text = text.replace("６", "6")
    text = text.replace("９", "9")
    text = text.replace("７", "7")
    text = text.replace("８", "8")
    text = text.replace("４", "4")
    text = re.sub(r"．\s*", ". ", text)
    text = text.replace("～", "~")
    text = text.replace("’", "'")
    text = text.replace("…", "...")
    text = text.replace("━", "-")
    text = text.replace("〈", "<")
    text = text.replace("〉", ">")
    text = text.replace("【", "[")
    text = text.replace("】", "]")
    text = text.replace("％", "%")
    return text


def remove_non_printing_char(text):
    
    output = []
    for char in text:
        cat = unicodedata.category(char)
        if cat.startswith("C"):
            continue
        output.append(char)
    return "".join(output)


def romanian_preprocessing(text):
    
    
    text = text.replace("\u015e", "\u0218").replace("\u015f", "\u0219")
    text = text.replace("\u0162", "\u021a").replace("\u0163", "\u021b")
    
    text = text.replace("\u0218", "S").replace("\u0219", "s")  
    text = text.replace("\u021a", "T").replace("\u021b", "t")  
    text = text.replace("\u0102", "A").replace("\u0103", "a")
    text = text.replace("\u00C2", "A").replace("\u00E2", "a")
    text = text.replace("\u00CE", "I").replace("\u00EE", "i")
    return text


class XLMTokenizer(PreTrainedTokenizer):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

    def __init__(
        self,
        vocab_file,
        merges_file,
        unk_token="<unk>",
        bos_token="<s>",
        sep_token="</s>",
        pad_token="<pad>",
        cls_token="</s>",
        mask_token="<special1>",
        additional_special_tokens=[
            "<special0>",
            "<special1>",
            "<special2>",
            "<special3>",
            "<special4>",
            "<special5>",
            "<special6>",
            "<special7>",
            "<special8>",
            "<special9>",
        ],
        lang2id=None,
        id2lang=None,
        do_lowercase_and_remove_accent=True,
        **kwargs,
    ):
        try:
            import sacremoses
        except ImportError:
            raise ImportError(
                "You need to install sacremoses to use XLMTokenizer. "
                "See https://pypi.org/project/sacremoses/ for installation."
            )

        self.sm = sacremoses

        
        self.cache_moses_punct_normalizer = {}
        
        self.cache_moses_tokenizer = {}
        self.lang_with_custom_tokenizer = {"zh", "th", "ja"}
        
        self.do_lowercase_and_remove_accent = do_lowercase_and_remove_accent
        self.lang2id = lang2id
        self.id2lang = id2lang
        if lang2id is not None and id2lang is not None:
            assert len(lang2id) == len(id2lang)

        self.ja_word_tokenizer = None
        self.zh_word_tokenizer = None

        with open(vocab_file, encoding="utf-8") as vocab_handle:
            self.encoder = json.load(vocab_handle)
        self.decoder = {v: k for k, v in self.encoder.items()}
        with open(merges_file, encoding="utf-8") as merges_handle:
            merges = merges_handle.read().split("\n")[:-1]
        merges = [tuple(merge.split()[:2]) for merge in merges]
        self.bpe_ranks = dict(zip(merges, range(len(merges))))
        self.cache = {}
        super().__init__(
            unk_token=unk_token,
            bos_token=bos_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            additional_special_tokens=additional_special_tokens,
            lang2id=lang2id,
            id2lang=id2lang,
            do_lowercase_and_remove_accent=do_lowercase_and_remove_accent,
            **kwargs,
        )

    @property
    def do_lower_case(self):
        return self.do_lowercase_and_remove_accent

    def moses_punct_norm(self, text, lang):
        if lang not in self.cache_moses_punct_normalizer:
            punct_normalizer = self.sm.MosesPunctNormalizer(lang=lang)
            self.cache_moses_punct_normalizer[lang] = punct_normalizer
        else:
            punct_normalizer = self.cache_moses_punct_normalizer[lang]
        return punct_normalizer.normalize(text)

    def moses_tokenize(self, text, lang):
        if lang not in self.cache_moses_tokenizer:
            moses_tokenizer = self.sm.MosesTokenizer(lang=lang)
            self.cache_moses_tokenizer[lang] = moses_tokenizer
        else:
            moses_tokenizer = self.cache_moses_tokenizer[lang]
        return moses_tokenizer.tokenize(text, return_str=False, escape=False)

    def moses_pipeline(self, text, lang):
        text = replace_unicode_punct(text)
        text = self.moses_punct_norm(text, lang)
        text = remove_non_printing_char(text)
        return text

    def ja_tokenize(self, text):
        if self.ja_word_tokenizer is None:
            try:
                import Mykytea

                self.ja_word_tokenizer = Mykytea.Mykytea(
                    f"-model {os.path.expanduser('~')}/local/share/kytea/model.bin"
                )
            except (AttributeError, ImportError):
                logger.error(
                    "Make sure you install KyTea (https://github.com/neubig/kytea) and it's python wrapper"
                    " (https://github.com/chezou/Mykytea-python) with the following steps"
                )
                logger.error("1. git clone git@github.com:neubig/kytea.git && cd kytea")
                logger.error("2. autoreconf -i")
                logger.error("3. ./configure --prefix=$HOME/local")
                logger.error("4. make && make install")
                logger.error("5. pip install kytea")
                raise
        return list(self.ja_word_tokenizer.getWS(text))

    @property
    def vocab_size(self):
        return len(self.encoder)

    def get_vocab(self):
        return dict(self.encoder, **self.added_tokens_encoder)

    def bpe(self, token):
        word = tuple(token[:-1]) + (token[-1] + "</w>",)
        if token in self.cache:
            return self.cache[token]
        pairs = get_pairs(word)

        if not pairs:
            return token + "</w>"

        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                except ValueError:
                    new_word.extend(word[i:])
                    break
                else:
                    new_word.extend(word[i:j])
                    i = j

                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
        word = " ".join(word)
        if word == "\n  </w>":
            word = "\n</w>"
        self.cache[token] = word
        return word

    def _tokenize(self, text, lang="en", bypass_tokenizer=False):
        
        if lang and self.lang2id and lang not in self.lang2id:
            logger.error(
                "Supplied language code not found in lang2id mapping. Please check that your language is supported by"
                " the loaded pretrained model."
            )
        if bypass_tokenizer:
            text = text.split()
        elif lang not in self.lang_with_custom_tokenizer:
            text = self.moses_pipeline(text, lang=lang)
            
            if lang == "ro":
                text = romanian_preprocessing(text)
            text = self.moses_tokenize(text, lang=lang)
        elif lang == "th":
            text = self.moses_pipeline(text, lang=lang)
            try:
                if "pythainlp" not in sys.modules:
                    from pythainlp.tokenize import word_tokenize as th_word_tokenize
                else:
                    th_word_tokenize = sys.modules["pythainlp"].word_tokenize
            except (AttributeError, ImportError):
                logger.error(
                    "Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps"
                )
                logger.error("1. pip install pythainlp")
                raise
            text = th_word_tokenize(text)
        elif lang == "zh":
            try:
                if "jieba" not in sys.modules:
                    import jieba
                else:
                    jieba = sys.modules["jieba"]
            except (AttributeError, ImportError):
                logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps")
                logger.error("1. pip install jieba")
                raise
            text = " ".join(jieba.cut(text))
            text = self.moses_pipeline(text, lang=lang)
            text = text.split()
        elif lang == "ja":
            text = self.moses_pipeline(text, lang=lang)
            text = self.ja_tokenize(text)
        else:
            raise ValueError("It should not reach here")

        if self.do_lowercase_and_remove_accent and not bypass_tokenizer:
            text = lowercase_and_remove_accent(text)

        split_tokens = []
        for token in text:
            if token:
                split_tokens.extend(list(self.bpe(token).split(" ")))

        return split_tokens

    def _convert_token_to_id(self, token):
        
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, index):
        
        return self.decoder.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens):
        
        out_string = "".join(tokens).replace("</w>", " ").strip()
        return out_string

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        bos = [self.bos_token_id]
        sep = [self.sep_token_id]

        if token_ids_1 is None:
            return bos + token_ids_0 + sep
        return bos + token_ids_0 + sep + token_ids_1 + sep

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        

        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        if token_ids_1 is not None:
            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
        return [1] + ([0] * len(token_ids_0)) + [1]

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )
        merge_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
        )

        with open(vocab_file, "w", encoding="utf-8") as f:
            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")

        index = 0
        with open(merge_file, "w", encoding="utf-8") as writer:
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
                if index != token_index:
                    logger.warning(
                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                        " Please check that the tokenizer is not corrupted!"
                    )
                    index = token_index
                writer.write(" ".join(bpe_tokens) + "\n")
                index += 1

        return vocab_file, merge_file

    def __getstate__(self):
        state = self.__dict__.copy()
        state["sm"] = None
        return state

    def __setstate__(self, d):
        self.__dict__ = d

        try:
            import sacremoses
        except ImportError:
            raise ImportError(
                "You need to install sacremoses to use XLMTokenizer. "
                "See https://pypi.org/project/sacremoses/ for installation."
            )

        self.sm = sacremoses