codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for seamlessm4t import os from shutil import copyfile from typing import any dict list optional tuple union import sentencepiece as spm from convertslowtokenizer import importprotobuf from tokenizationutils import batchencoding pretokenizedinput pretrainedtokenizer textinput from tokenizationutilsbase import addedtoken from utils import paddingstrategy logging logger logging getloggername pretrainedvocabfilesmap vocabfile facebookhfseamlessm4tmedium https huggingface cofacebookhfseamlessm4tmediumblobmainsentencepiece bpe model spieceunderline vocabfilesnames vocabfile sentencepiece bpe model pretrainedpositionalembeddingssizes facebookhfseamlessm4tmedium 2048 class seamlessm4ttokenizerpretrainedtokenizer vocabfilesnames vocabfilesnames maxmodelinputsizes pretrainedpositionalembeddingssizes pretrainedvocabfilesmap pretrainedvocabfilesmap modelinputnames inputids attentionmask prefixtokens listint suffixtokens listint def init self vocabfile bostokens eostokens septokens clstokens unktokenunk padtokenpad tokenizerfilenone srclangeng tgtlangfra spmodelkwargs optionaldictstr any none additionalspecialtokensnone kwargs self spmodelkwargs if spmodelkwargs is none else spmodelkwargs add this unused argument to keep some important copied from statements self legacy false self vocabfile vocabfile self spmodel self getspmprocessorkwargs popfromslow false vocab 0 1 2 3 4 5 6 7 8 9 spm unk s s an en d er in s a fairseq pad unk s s an en d er in s mimic fairseq tokentoid alignment for the first 4 token self addedtokensdecoder 0 addedtokenpadtoken specialtrue if isinstancepadtoken str else padtoken 1 addedtokenunktoken specialtrue if isinstanceunktoken str else unktoken 2 addedtokenbostoken specialtrue if isinstancebostoken str else bostoken 3 addedtokeneostoken specialtrue if isinstanceeostoken str else eostoken the first real token an has position 4 in the original fairseq vocab and position 3 in the spm vocab self fairseqoffset 1 self spmodelsize lenself spmodel self srclang fsrclang if not in srclang else srclang self tgtlang ftgtlang if not in tgtlang else tgtlang super init bostokenbostoken eostokeneostoken unktokenunktoken septokenseptoken clstokenclstoken padtokenpadtoken tokenizerfiletokenizerfile srclangsrclang tgtlangtgtlang additionalspecialtokensadditionalspecialtokens spmodelkwargsself spmodelkwargs kwargs self setsrclangspecialtokensself srclang self settgtlangspecialtokensself tgtlang copied from transformers models nllb tokenizationnllb nllbtokenizer getstate def getstateself state self dict copy statespmodel none statespmodelproto self spmodel serializedmodelproto return state copied from transformers models nllb tokenizationnllb nllbtokenizer setstate def setstateself d self dict d for backward compatibility if not hasattrself spmodelkwargs self spmodelkwargs self spmodel spm sentencepieceprocessorself spmodelkwargs self spmodel loadfromserializedprotoself spmodelproto property def vocabsizeself return lenself spmodel def call self text uniontextinput pretokenizedinput listtextinput listpretokenizedinput none textpair optionaluniontextinput pretokenizedinput listtextinput listpretokenizedinput none texttarget uniontextinput pretokenizedinput listtextinput listpretokenizedinput none textpairtarget optional uniontextinput pretokenizedinput listtextinput listpretokenizedinput none padding unionbool str paddingstrategy true padtomultipleof optionalint 2 srclang optionalstr none tgtlang optionalstr none kwargs if srclang is not none self srclang srclang if tgtlang is not none self tgtlang tgtlang output super call texttext textpairtextpair texttargettexttarget textpairtargettextpairtarget paddingpadding padtomultipleofpadtomultipleof kwargs return batchencodingoutput tensortypekwargs getreturntensors property copied from transformers models nllb tokenizationnllb nllbtokenizer srclang def srclangself str return self srclang srclang setter def srclangself newsrclang str none if not in newsrclang self srclang fnewsrclang else self srclang newsrclang self setsrclangspecialtokensself srclang property def tgtlangself str return self tgtlang tgtlang setter def tgtlangself newtgtlang str none if not in newtgtlang self tgtlang fnewtgtlang else self tgtlang newtgtlang self settgtlangspecialtokensself tgtlang copied from transformers models nllb tokenizationnllb nllbtokenizer getspecialtokensmask def getspecialtokensmask self tokenids0 listint tokenids1 optionallistint none alreadyhasspecialtokens bool false listint if alreadyhasspecialtokens return super getspecialtokensmask tokenids0tokenids0 tokenids1tokenids1 alreadyhasspecialtokenstrue prefixones 1 lenself prefixtokens suffixones 1 lenself suffixtokens if tokenids1 is none return prefixones 0 lentokenids0 suffixones return prefixones 0 lentokenids0 0 lentokenids1 suffixones copied from transformers models nllb tokenizationnllb nllbtokenizer buildinputswithspecialtokens def buildinputswithspecialtokens self tokenids0 listint tokenids1 optionallistint none listint if tokenids1 is none return self prefixtokens tokenids0 self suffixtokens we don t expect to process pairs but leave the pair logic for api consistency return self prefixtokens tokenids0 tokenids1 self suffixtokens copied from transformers models nllb tokenizationnllb nllbtokenizer createtokentypeidsfromsequences def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint sep self septokenid cls self clstokenid if tokenids1 is none return lencls tokenids0 sep 0 return lencls tokenids0 sep sep tokenids1 sep 0 def buildtranslationinputs self rawinputs returntensors str srclang optionalstr tgtlang optionalstr extrakwargs copied from transformers models t5 tokenizationt5 t5tokenizer getspmprocessor copied from transformers models t5 tokenizationt5 t5tokenizer tokenize converts a string to a list of tokens if self legacy is set to false a prefix token is added unless the first token is special copied from transformers models t5 tokenizationt5 t5tokenizer tokenize returns a tokenized string we deactivated the adddummyprefix option thus the sentencepiece internals will always strip any spieceunderline for example self spmodel encodefspieceunderlinehey outtype str will give h e y instead of he y thus we always encode funktokentext and strip the unktoken here is an example with unktoken unk and unktokenlength 4 self tokenizer spmodel encodeunk hey outtype str4 1 encode string prefix ex unk hey 2 remove self unktoken from unk hey converts a token str in an id using the vocab spmid self spmodel piecetoidtoken need to return unknown token if the sp model returned 0 return spmid self fairseqoffset if spmid else self unktokenid def convertidtotokenself index converts a sequence of tokens strings for subwords in a single string if tokens0 startswithspieceunderline tokens0 tokens01 outstring jointokens replacespieceunderline strip return outstring copied from transformers models nllb tokenizationnllb nllbtokenizer savevocabulary def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr if not os path isdirsavedirectory logger errorfvocabulary path savedirectory should be a directory return outvocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesvocabfile if os path abspathself vocabfile os path abspathoutvocabfile and os path isfileself vocabfile copyfileself vocabfile outvocabfile elif not os path isfileself vocabfile with openoutvocabfile wb as fi contentspiecemodel self spmodel serializedmodelproto fi writecontentspiecemodel return outvocabfile copied from transformers models nllb tokenizationnllb nllbtokenizer prepareseq2seqbatch with englatneng fralatnfra def prepareseq2seqbatch self srctexts liststr srclang str eng tgttexts optionalliststr none tgtlang str fra kwargs batchencoding self srclang srclang self tgtlang tgtlang return super prepareseq2seqbatchsrctexts tgttexts kwargs copied from transformers models nllb tokenizationnllb nllbtokenizer switchtoinputmode def switchtoinputmodeself return self setsrclangspecialtokensself srclang copied from transformers models nllb tokenizationnllb nllbtokenizer switchtotargetmode def switchtotargetmodeself return self settgtlangspecialtokensself tgtlang def setsrclangspecialtokensself srclang none self curlangcode self converttokenstoidssrclang self initkwargssrclang srclang if self curlangcode self unktokenid logger warningonce fsrclangsrclang has not be found in the vocabulary behaviour will probably be unexpected because the language token id will be replaced by the unknown token id self prefixtokens self curlangcode self suffixtokens self eostokenid https github comfacebookresearchfairseq2blobc53f18e6be6b8b46b722f2249b8397b7eccd7ad3srcfairseq2modelsnllbtokenizer pyl112l116 def settgtlangspecialtokensself lang str none self curlangcode self converttokenstoidslang self initkwargstgtlang lang if self curlangcode self unktokenid logger warningonce ftgtlanglang has not be found in the vocabulary behaviour will probably be unexpected because the language token id will be replaced by the unknown token id self prefixtokens self eostokenid self curlangcode self suffixtokens self eostokenid coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for seamlessm4t construct a seamlessm4t tokenizer adapted from robertatokenizer and xlnettokenizer based on sentencepiece https github com google sentencepiece the tokenization method is language code tokens eos for source language documents and eos language code tokens eos for target language documents examples python from transformers import seamlessm4ttokenizer tokenizer seamlessm4ttokenizer from_pretrained facebook hf seamless m4t medium src_lang eng tgt_lang fra example_english_phrase un chief says there is no military solution in syria expected_translation_french le chef de l onu affirme qu il n y a pas de solution militaire en syrie inputs tokenizer example_english_phrase text_target expected_translation_french return_tensors pt args vocab_file str path to the vocabulary file bos_token str optional defaults to s the beginning of sequence token that was used during pretraining can be used a sequence classifier token tip when building a sequence using special tokens this is not the token that is used for the beginning of sequence the token used is the cls_token tip eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip sep_token str optional defaults to s the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens cls_token str optional defaults to s the classifier token which is used when doing sequence classification classification of the whole sequence instead of per token classification it is the first token of the sequence when built with special tokens unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths tokenizer_file str optional the path to a tokenizer file to use instead of the vocab file src_lang str optional defaults to eng the language to use as source language for translation tgt_lang str optional defaults to fra the language to use as target language for translation sp_model_kwargs dict str any optional additional keyword arguments to pass to the model initialization additional_special_tokens tuple or list of str or tokenizers addedtoken optional a tuple or a list of additional special tokens can be used to specify the list of languages that will be supported by the tokenizer add this unused argument to keep some important copied from statements vocab 0 1 2 3 4 5 6 7 8 9 spm unk s s an en _d er in _s _a fairseq pad unk s s an en d er in s mimic fairseq token to id alignment for the first 4 token the first real token an has position 4 in the original fairseq vocab and position 3 in the spm vocab copied from transformers models nllb tokenization_nllb nllbtokenizer __getstate__ copied from transformers models nllb tokenization_nllb nllbtokenizer __setstate__ for backward compatibility args text str list str list list str optional the sequence or batch of sequences to be encoded each sequence can be a string or a list of strings pretokenized string if the sequences are provided as list of strings pretokenized you must set is_split_into_words true to lift the ambiguity with a batch of sequences text_pair str list str list list str optional the sequence or batch of sequences to be encoded each sequence can be a string or a list of strings pretokenized string if the sequences are provided as list of strings pretokenized you must set is_split_into_words true to lift the ambiguity with a batch of sequences text_target str list str list list str optional the sequence or batch of sequences to be encoded as target texts each sequence can be a string or a list of strings pretokenized string if the sequences are provided as list of strings pretokenized you must set is_split_into_words true to lift the ambiguity with a batch of sequences text_pair_target str list str list list str optional the sequence or batch of sequences to be encoded as target texts each sequence can be a string or a list of strings pretokenized string if the sequences are provided as list of strings pretokenized you must set is_split_into_words true to lift the ambiguity with a batch of sequences padding bool str or utils paddingstrategy optional defaults to true select a strategy to pad the returned sequences according to the model s padding side and padding index among true or longest pad to the longest sequence in the batch or no padding if only a single sequence if provided max_length pad to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided false or do_not_pad default no padding i e can output a batch with sequences of different lengths pad_to_multiple_of int optional if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta src_lang str optional a string representing the source language if not specified the last src_lang specified either during initialization or when calling this tokenizer will be used tgt_lang str optional a string representing the target language if not specified the last tgt_lang specified either during initialization or when calling this tokenizer will be used kwargs optional remaining dictionary of keyword arguments that will be passed to pretrainedtokenizer __call__ copied from transformers models nllb tokenization_nllb nllbtokenizer src_lang copied from transformers models nllb tokenization_nllb nllbtokenizer get_special_tokens_mask retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepare_for_model method args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs already_has_special_tokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns list int a list of integers in the range 0 1 1 for a special token 0 for a sequence token copied from transformers models nllb tokenization_nllb nllbtokenizer build_inputs_with_special_tokens build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens an nllb sequence has the following format where x represents the sequence input_ids for encoder x eos src_lang_code decoder_input_ids for decoder x eos tgt_lang_code bos is never used pairs of sequences are not the expected use case but they will be handled without a separator args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens we don t expect to process pairs but leave the pair logic for api consistency copied from transformers models nllb tokenization_nllb nllbtokenizer create_token_type_ids_from_sequences create a mask from the two sequences passed to be used in a sequence pair classification task nllb does not make use of token type ids therefore a list of zeros is returned args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of zeros used by translation pipeline to prepare inputs for the generate function copied from transformers models t5 tokenization_t5 t5tokenizer get_spm_processor no dependency on protobuf copied from transformers models t5 tokenization_t5 t5tokenizer tokenize converts a string to a list of tokens if self legacy is set to false a prefix token is added unless the first token is special copied from transformers models t5 tokenization_t5 t5tokenizer _tokenize returns a tokenized string we de activated the add_dummy_prefix option thus the sentencepiece internals will always strip any spiece_underline for example self sp_model encode f spiece_underline hey out_type str will give h e y instead of he y thus we always encode f unk_token text and strip the unk_token here is an example with unk_token unk and unk_token_length 4 self tokenizer sp_model encode unk hey out_type str 4 1 encode string prefix ex unk hey 2 remove self unk_token from unk hey converts a token str in an id using the vocab need to return unknown token if the sp model returned 0 converts an index integer in a token str using the vocab converts a sequence of tokens strings for sub words in a single string copied from transformers models nllb tokenization_nllb nllbtokenizer save_vocabulary copied from transformers models nllb tokenization_nllb nllbtokenizer prepare_seq2seq_batch with eng_latn eng fra_latn fra copied from transformers models nllb tokenization_nllb nllbtokenizer _switch_to_input_mode copied from transformers models nllb tokenization_nllb nllbtokenizer _switch_to_target_mode reset the special tokens to the source lang setting prefix src_lang_code suffix eos https github com facebookresearch fairseq2 blob c53f18e6be6b8b46b722f2249b8397b7eccd7ad3 src fairseq2 models nllb tokenizer py l112 l116 reset the special tokens to the target lang setting prefix eos tgt_lang_code and suffix eos
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...convert_slow_tokenizer import import_protobuf
from ...tokenization_utils import (
    BatchEncoding,
    PreTokenizedInput,
    PreTrainedTokenizer,
    TextInput,
)
from ...tokenization_utils_base import AddedToken
from ...utils import PaddingStrategy, logging


logger = logging.get_logger(__name__)

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "facebook/hf-seamless-m4t-medium": (
            "https://huggingface.co/facebook/hf-seamless-m4t-medium/blob/main/sentencepiece.bpe.model"
        ),
    }
}

SPIECE_UNDERLINE = "▁"


VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.bpe.model"}


PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "facebook/hf-seamless-m4t-medium": 2048,
}


class SeamlessM4TTokenizer(PreTrainedTokenizer):
    

    vocab_files_names = VOCAB_FILES_NAMES
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    model_input_names = ["input_ids", "attention_mask"]

    prefix_tokens: List[int] = []
    suffix_tokens: List[int] = []

    def __init__(
        self,
        vocab_file,
        bos_token="<s>",
        eos_token="</s>",
        sep_token="</s>",
        cls_token="<s>",
        unk_token="<unk>",
        pad_token="<pad>",
        tokenizer_file=None,
        src_lang="eng",
        tgt_lang="fra",
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        additional_special_tokens=None,
        **kwargs,
    ):
        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
        
        self.legacy = False
        self.vocab_file = vocab_file

        self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))

        
        
        
        

        
        self._added_tokens_decoder = {
            0: AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token,
            1: AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token,
            2: AddedToken(bos_token, special=True) if isinstance(bos_token, str) else bos_token,
            3: AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token,
        }

        
        self.fairseq_offset = 1

        self.sp_model_size = len(self.sp_model)

        self._src_lang = f"__{src_lang}__" if "__" not in src_lang else src_lang
        self._tgt_lang = f"__{tgt_lang}__" if "__" not in tgt_lang else tgt_lang

        super().__init__(
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            sep_token=sep_token,
            cls_token=cls_token,
            pad_token=pad_token,
            tokenizer_file=tokenizer_file,
            src_lang=src_lang,
            tgt_lang=tgt_lang,
            additional_special_tokens=additional_special_tokens,
            sp_model_kwargs=self.sp_model_kwargs,
            **kwargs,
        )

        self.set_src_lang_special_tokens(self._src_lang)
        self.set_tgt_lang_special_tokens(self._tgt_lang)

    
    def __getstate__(self):
        state = self.__dict__.copy()
        state["sp_model"] = None
        state["sp_model_proto"] = self.sp_model.serialized_model_proto()
        return state

    
    def __setstate__(self, d):
        self.__dict__ = d

        
        if not hasattr(self, "sp_model_kwargs"):
            self.sp_model_kwargs = {}

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.LoadFromSerializedProto(self.sp_model_proto)

    @property
    def vocab_size(self):
        return len(self.sp_model)

    def __call__(
        self,
        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
        text_pair: Optional[Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]] = None,
        text_target: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
        text_pair_target: Optional[
            Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]
        ] = None,
        padding: Union[bool, str, PaddingStrategy] = True,
        pad_to_multiple_of: Optional[int] = 2,
        src_lang: Optional[str] = None,
        tgt_lang: Optional[str] = None,
        **kwargs,
    ):
        
        if src_lang is not None:
            self.src_lang = src_lang
        if tgt_lang is not None:
            self.tgt_lang = tgt_lang

        output = super().__call__(
            text=text,
            text_pair=text_pair,
            text_target=text_target,
            text_pair_target=text_pair_target,
            padding=padding,
            pad_to_multiple_of=pad_to_multiple_of,
            **kwargs,
        )

        return BatchEncoding(output, tensor_type=kwargs.get("return_tensors"))

    @property
    
    def src_lang(self) -> str:
        return self._src_lang

    @src_lang.setter
    def src_lang(self, new_src_lang: str) -> None:
        if "__" not in new_src_lang:
            self._src_lang = f"__{new_src_lang}__"
        else:
            self._src_lang = new_src_lang
        self.set_src_lang_special_tokens(self._src_lang)

    @property
    def tgt_lang(self) -> str:
        return self._tgt_lang

    @tgt_lang.setter
    def tgt_lang(self, new_tgt_lang: str) -> None:
        if "__" not in new_tgt_lang:
            self._tgt_lang = f"__{new_tgt_lang}__"
        else:
            self._tgt_lang = new_tgt_lang
        self.set_tgt_lang_special_tokens(self._tgt_lang)

    
    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        

        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        prefix_ones = [1] * len(self.prefix_tokens)
        suffix_ones = [1] * len(self.suffix_tokens)
        if token_ids_1 is None:
            return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
        return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones

    
    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        if token_ids_1 is None:
            return self.prefix_tokens + token_ids_0 + self.suffix_tokens
        
        return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens

    
    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        

        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

    def _build_translation_inputs(
        self, raw_inputs, return_tensors: str, src_lang: Optional[str], tgt_lang: Optional[str], **extra_kwargs
    ):
        
        if src_lang is None or tgt_lang is None:
            raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model.")
        self.src_lang = src_lang
        inputs = self(raw_inputs, add_special_tokens=True, return_tensors=return_tensors, **extra_kwargs)
        if "__" not in tgt_lang:
            tgt_lang = f"__{tgt_lang}__"
        tgt_lang_id = self.convert_tokens_to_ids(tgt_lang)
        inputs["forced_bos_token_id"] = tgt_lang_id
        return inputs

    def get_vocab(self):
        vocab = {
            self.convert_ids_to_tokens(i): i for i in range(self.fairseq_offset, self.vocab_size + self.fairseq_offset)
        }
        vocab.update(self.added_tokens_encoder)
        return vocab

    @property
    def unk_token_length(self):
        return len(self.sp_model.encode(str(self.unk_token)))

    
    def get_spm_processor(self, from_slow=False):
        tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        if self.legacy or from_slow:  
            tokenizer.Load(self.vocab_file)
            return tokenizer

        with open(self.vocab_file, "rb") as f:
            sp_model = f.read()
            model_pb2 = import_protobuf(f"The new behaviour of {self.__class__.__name__} (with `self.legacy = False`)")
            model = model_pb2.ModelProto.FromString(sp_model)
            normalizer_spec = model_pb2.NormalizerSpec()
            normalizer_spec.add_dummy_prefix = False
            model.normalizer_spec.MergeFrom(normalizer_spec)
            sp_model = model.SerializeToString()
            tokenizer.LoadFromSerializedProto(sp_model)
        return tokenizer

    
    def tokenize(self, text: "TextInput", add_special_tokens=False, **kwargs) -> List[str]:
        
        if self.legacy or len(text) == 0:
            return super().tokenize(text, **kwargs)

        tokens = super().tokenize(SPIECE_UNDERLINE + text.replace(SPIECE_UNDERLINE, " "), **kwargs)

        if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:
            tokens = tokens[1:]
        return tokens

    
    def _tokenize(self, text, **kwargs):
        
        tokens = self.sp_model.encode(text, out_type=str)
        if self.legacy or not text.startswith((SPIECE_UNDERLINE, " ")):
            return tokens

        
        tokens = self.sp_model.encode(self.unk_token + text, out_type=str)
        
        return tokens[self.unk_token_length :] if len(tokens) >= self.unk_token_length else tokens

    def _convert_token_to_id(self, token):
        
        spm_id = self.sp_model.PieceToId(token)

        
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

    def _convert_id_to_token(self, index):
        
        return self.sp_model.IdToPiece(index - self.fairseq_offset)

    def convert_tokens_to_string(self, tokens):
        
        if tokens[0].startswith(SPIECE_UNDERLINE):
            tokens[0] = tokens[0][1:]

        out_string = "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()
        return out_string

    
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        return (out_vocab_file,)

    
    def prepare_seq2seq_batch(
        self,
        src_texts: List[str],
        src_lang: str = "eng",
        tgt_texts: Optional[List[str]] = None,
        tgt_lang: str = "fra",
        **kwargs,
    ) -> BatchEncoding:
        self.src_lang = src_lang
        self.tgt_lang = tgt_lang
        return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs)

    
    def _switch_to_input_mode(self):
        return self.set_src_lang_special_tokens(self.src_lang)

    
    def _switch_to_target_mode(self):
        return self.set_tgt_lang_special_tokens(self.tgt_lang)

    def set_src_lang_special_tokens(self, src_lang) -> None:
        
        self.cur_lang_code = self.convert_tokens_to_ids(src_lang)
        self.init_kwargs["src_lang"] = src_lang

        if self.cur_lang_code == self.unk_token_id:
            logger.warning_once(
                f"`src_lang={src_lang}` has not be found in the vocabulary. Behaviour will probably be unexpected because the language token id will be replaced by the unknown token id."
            )

        self.prefix_tokens = [self.cur_lang_code]
        self.suffix_tokens = [self.eos_token_id]

    
    def set_tgt_lang_special_tokens(self, lang: str) -> None:
        
        self.cur_lang_code = self.convert_tokens_to_ids(lang)
        self.init_kwargs["tgt_lang"] = lang

        if self.cur_lang_code == self.unk_token_id:
            logger.warning_once(
                f"`tgt_lang={lang}` has not be found in the vocabulary. Behaviour will probably be unexpected because the language token id will be replaced by the unknown token id."
            )

        self.prefix_tokens = [self.eos_token_id, self.cur_lang_code]
        self.suffix_tokens = [self.eos_token_id]