codingutf8 2018 t5 s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model t5 import os import re import warnings from shutil import copyfile from typing import typechecking any dict list optional tuple import sentencepiece as spm from convertslowtokenizer import importprotobuf from tokenizationutils import pretrainedtokenizer from tokenizationutilsbase import addedtoken if typechecking from tokenizationutilsbase import textinput from utils import logging logger logging getloggername vocabfilesnames vocabfile spiece model pretrainedvocabfilesmap vocabfile t5small https huggingface cot5smallresolvemainspiece model t5base https huggingface cot5baseresolvemainspiece model t5large https huggingface cot5largeresolvemainspiece model t53b https huggingface cot53bresolvemainspiece model t511b https huggingface cot511bresolvemainspiece model todopvp this should be removed in transformers v5 pretrainedpositionalembeddingssizes t5small 512 t5base 512 t5large 512 t53b 512 t511b 512 spieceunderline class t5tokenizerpretrainedtokenizer vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask def init self vocabfile eostokens unktokenunk padtokenpad extraids100 additionalspecialtokensnone spmodelkwargs optionaldictstr any none legacynone kwargs none padtoken addedtokenpadtoken specialtrue if isinstancepadtoken str else padtoken unktoken addedtokenunktoken specialtrue if isinstanceunktoken str else unktoken eostoken addedtokeneostoken specialtrue if isinstanceeostoken str else eostoken self spmodelkwargs if spmodelkwargs is none else spmodelkwargs self vocabfile vocabfile self extraids extraids self spmodel spm sentencepieceprocessorself spmodelkwargs self spmodel loadvocabfile if additionalspecialtokens is not none extratokens x for x in additionalspecialtokens if extraid in strx if lenextratokens 1 additionalspecialtokens fextraidi for i in rangeextraids elif extraids 0 and extraids lenextratokens raise valueerror fboth extraids extraids and additionalspecialtokens additionalspecialtokens are provided to t5tokenizer in this case the additionalspecialtokens must include the extraids tokens else extratokens fextraidi for i in rangeextraids additionalspecialtokens extratokens for legacy purpose we keep this will be removed and tests updated when addedtokensdecoder is not passed as kwargs self addedtokensdecoder for i in rangelenextratokens self addedtokensdecoderlenself spmodel 1 extraids i addedtoken fextraidi singlewordtrue lstriptrue rstriptrue specialtrue if legacy is none logger warningonce fyou are using the default legacy behaviour of the self class this is expected and simply means that the legacy previous behavior will be used so nothing changes for you if you want to use the new behaviour set legacyfalse this should only be set if you understand what it means and thouroughly read the reason why this was added as explained in https github comhuggingfacetransformerspull24565 legacy true self legacy legacy self spmodel self getspmprocessorkwargs popfromslow false self vocabfile vocabfile self extraids extraids super init eostokeneostoken unktokenunktoken padtokenpadtoken extraidsextraids additionalspecialtokensadditionalspecialtokens spmodelkwargsself spmodelkwargs legacylegacy kwargs copied from transformers models t5 tokenizationt5 t5tokenizer getspmprocessor def getspmprocessorself fromslowfalse tokenizer spm sentencepieceprocessorself spmodelkwargs if self legacy or fromslow no dependency on protobuf tokenizer loadself vocabfile return tokenizer with openself vocabfile rb as f spmodel f read modelpb2 importprotobuffthe new behaviour of self class name with self legacy false model modelpb2 modelproto fromstringspmodel normalizerspec modelpb2 normalizerspec normalizerspec adddummyprefix false model normalizerspec mergefromnormalizerspec spmodel model serializetostring tokenizer loadfromserializedprotospmodel return tokenizer staticmethod def eventuallycorrectt5maxlengthpretrainedmodelnameorpath maxmodellength initmaxmodellength if pretrainedmodelnameorpath in t5tokenizer maxmodelinputsizes deprecatedmaxmodellength t5tokenizer maxmodelinputsizespretrainedmodelnameorpath if initmaxmodellength is not none and initmaxmodellength maxmodellength return initmaxmodellength elif initmaxmodellength is none warnings warn this tokenizer was incorrectly instantiated with a model max length of f deprecatedmaxmodellength which will be corrected in transformers v5 nfor now this behavior is kept to avoid breaking backwards compatibility when paddingencoding with truncation is true n be aware that you should not rely on f pretrainedmodelnameorpath automatically truncating your input to f deprecatedmaxmodellength when paddingencoding n if you want to encodepad to sequences f longer than deprecatedmaxmodellength you can either instantiate this tokenizer with modelmaxlength or pass maxlength when encodingpadding n to avoid this warning please instantiate this tokenizer with modelmaxlength set to your preferred value futurewarning return maxmodellength property def vocabsizeself return self spmodel getpiecesize def getvocabself vocab self convertidstotokensi i for i in rangeself vocabsize vocab updateself addedtokensencoder return vocab def getspecialtokensmask self tokenids0 listint tokenids1 optionallistint none alreadyhasspecialtokens bool false listint if alreadyhasspecialtokens return super getspecialtokensmask tokenids0tokenids0 tokenids1tokenids1 alreadyhasspecialtokenstrue normal case some special tokens if tokenids1 is none return 0 lentokenids0 1 return 0 lentokenids0 1 0 lentokenids1 1 def getsentineltokensself return list setfilterlambda x boolre searchrextraidd x is not none self additionalspecialtokens def getsentineltokenidsself return self converttokenstoidstoken for token in self getsentineltokens def addeosifnotpresentself tokenids listint listint create a mask from the two sequences passed to be used in a sequencepair classification task t5 does not make use of token type ids therefore a list of zeros is returned args tokenids0 listint list of ids tokenids1 listint optional optional second list of ids for sequence pairs returns listint list of zeros build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a sequence has the following format single sequence x s pair of sequences a s b s args tokenids0 listint list of ids to which the special tokens will be added tokenids1 listint optional optional second list of ids for sequence pairs returns listint list of input ids glossaryinputids with the appropriate special tokens for backward compatibility copied from transformers models t5 tokenizationt5 t5tokenizer tokenize converts a string to a list of tokens if self legacy is set to false a prefix token is added unless the first token is special returns a tokenized string we deactivated the adddummyprefix option thus the sentencepiece internals will always strip any spieceunderline for example self spmodel encodefspieceunderlinehey outtype str will give h e y instead of he y thus we always encode funktokentext and strip the unktoken here is an example with unktoken unk and unktokenlength 4 self tokenizer spmodel encodeunk hey outtype str4 1 encode string prefix ex unk hey 2 remove self unktoken from unk hey converts a token str in an id using the vocab return self spmodel piecetoidtoken def convertidtotokenself index converts a sequence of tokens string in a single string currentsubtokens since we manually add the prefix space we have to remove it tokens0 tokens0 lstripspieceunderline outstring previsspecial false for token in tokens make sure that special tokens are not decoded using sentencepiece model if token in self allspecialtokens if not previsspecial outstring outstring self spmodel decodecurrentsubtokens token previsspecial true currentsubtokens else currentsubtokens appendtoken previsspecial false outstring self spmodel decodecurrentsubtokens return outstring strip def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr if not os path isdirsavedirectory logger errorfvocabulary path savedirectory should be a directory return outvocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesvocabfile if os path abspathself vocabfile os path abspathoutvocabfile and os path isfileself vocabfile copyfileself vocabfile outvocabfile elif not os path isfileself vocabfile with openoutvocabfile wb as fi contentspiecemodel self spmodel serializedmodelproto fi writecontentspiecemodel return outvocabfile coding utf 8 2018 t5 s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model t5 todo pvp this should be removed in transformers v5 construct a t5 tokenizer based on sentencepiece https github com google sentencepiece this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str sentencepiece https github com google sentencepiece file generally has a spm extension that contains the vocabulary necessary to instantiate a tokenizer eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths extra_ids int optional defaults to 100 add a number of extra ids added to the vocabulary for use as sentinels these tokens are accessible as extra_id_ d where d is a number between 0 and extra_ids 1 these tokens can be retrieved by calling get_sentinel_tokens method and token ids can be by calling get_sentinel_token_ids method additional_special_tokens list str optional additional special tokens used by the tokenizer sp_model_kwargs dict optional will be passed to the sentencepieceprocessor __init__ method the python wrapper for sentencepiece https github com google sentencepiece tree master python can be used among other things to set enable_sampling enable subword regularization nbest_size sampling parameters for unigram invalid for bpe dropout nbest_size 0 1 no sampling is performed nbest_size 1 samples from the nbest_size results nbest_size 0 assuming that nbest_size is infinite and samples from the all hypothesis lattice using forward filtering and backward sampling algorithm alpha smoothing parameter for unigram sampling and dropout probability of merge operations for bpe dropout legacy bool optional whether or not the legacy behaviour of the tokenizer should be used legacy is before the merge of 24622 and 25224 which includes fixes to properly handle tokens that appear after special tokens a simple example legacy true python from transformers import t5tokenizer tokenizer t5tokenizer from_pretrained t5 base legacy true tokenizer encode hello extra_id_0 8774 32099 3 5 1 legacy false python from transformers import t5tokenizer tokenizer t5tokenizer from_pretrained t5 base legacy false tokenizer encode hello extra_id_0 the extra space 3 is no longer here 8774 32099 5 1 checkout the pull request https github com huggingface transformers pull 24565 for more details attributes sp_model sentencepieceprocessor the sentencepiece processor that is used for every conversion string tokens and ids for legacy purpose we keep this will be removed and tests updated when added_tokens_decoder is not passed as kwargs copied from transformers models t5 tokenization_t5 t5tokenizer get_spm_processor no dependency on protobuf retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepare_for_model method args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs already_has_special_tokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns list int a list of integers in the range 0 1 1 for a special token 0 for a sequence token normal case some special tokens do not add eos again if user already added it create a mask from the two sequences passed to be used in a sequence pair classification task t5 does not make use of token type ids therefore a list of zeros is returned args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of zeros build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a sequence has the following format single sequence x s pair of sequences a s b s args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens for backward compatibility copied from transformers models t5 tokenization_t5 t5tokenizer tokenize converts a string to a list of tokens if self legacy is set to false a prefix token is added unless the first token is special returns a tokenized string we de activated the add_dummy_prefix option thus the sentencepiece internals will always strip any spiece_underline for example self sp_model encode f spiece_underline hey out_type str will give h e y instead of he y thus we always encode f unk_token text and strip the unk_token here is an example with unk_token unk and unk_token_length 4 self tokenizer sp_model encode unk hey out_type str 4 1 encode string prefix ex unk hey 2 remove self unk_token from unk hey converts a token str in an id using the vocab converts an index integer in a token str using the vocab converts a sequence of tokens string in a single string since we manually add the prefix space we have to remove it make sure that special tokens are not decoded using sentencepiece model
import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...convert_slow_tokenizer import import_protobuf
from ...tokenization_utils import PreTrainedTokenizer
from ...tokenization_utils_base import AddedToken


if TYPE_CHECKING:
    from ...tokenization_utils_base import TextInput
from ...utils import logging


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/spiece.model",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/spiece.model",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/spiece.model",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/spiece.model",
    }
}



PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "t5-small": 512,
    "t5-base": 512,
    "t5-large": 512,
    "t5-3b": 512,
    "t5-11b": 512,
}

SPIECE_UNDERLINE = "▁"


class T5Tokenizer(PreTrainedTokenizer):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        eos_token="</s>",
        unk_token="<unk>",
        pad_token="<pad>",
        extra_ids=100,
        additional_special_tokens=None,
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        legacy=None,
        **kwargs,
    ) -> None:
        pad_token = AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token
        unk_token = AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token
        eos_token = AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token

        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs

        self.vocab_file = vocab_file
        self._extra_ids = extra_ids

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(vocab_file)

        if additional_special_tokens is not None:
            extra_tokens = [x for x in additional_special_tokens if "<extra_id_" in str(x)]
            if len(extra_tokens) < 1:
                additional_special_tokens += [f"<extra_id_{i}>" for i in range(extra_ids)]
            elif extra_ids > 0 and extra_ids != len(extra_tokens):
                raise ValueError(
                    f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
                    " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
                    " tokens"
                )
        else:
            extra_tokens = [f"<extra_id_{i}>" for i in range(extra_ids)]
            additional_special_tokens = extra_tokens

        
        self._added_tokens_decoder = {}
        for i in range(len(extra_tokens)):
            self._added_tokens_decoder[len(self.sp_model) - 1 + extra_ids - i] = AddedToken(
                f"<extra_id_{i}>", single_word=True, lstrip=True, rstrip=True, special=True
            )

        if legacy is None:
            logger.warning_once(
                f"You are using the default legacy behaviour of the {self.__class__}. This is"
                " expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you."
                " If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it"
                " means, and thouroughly read the reason why this was added as explained in"
                " https://github.com/huggingface/transformers/pull/24565"
            )
            legacy = True

        self.legacy = legacy
        self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
        self.vocab_file = vocab_file
        self._extra_ids = extra_ids

        super().__init__(
            eos_token=eos_token,
            unk_token=unk_token,
            pad_token=pad_token,
            extra_ids=extra_ids,
            additional_special_tokens=additional_special_tokens,
            sp_model_kwargs=self.sp_model_kwargs,
            legacy=legacy,
            **kwargs,
        )

    
    def get_spm_processor(self, from_slow=False):
        tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        if self.legacy or from_slow:  
            tokenizer.Load(self.vocab_file)
            return tokenizer

        with open(self.vocab_file, "rb") as f:
            sp_model = f.read()
            model_pb2 = import_protobuf(f"The new behaviour of {self.__class__.__name__} (with `self.legacy = False`)")
            model = model_pb2.ModelProto.FromString(sp_model)
            normalizer_spec = model_pb2.NormalizerSpec()
            normalizer_spec.add_dummy_prefix = False
            model.normalizer_spec.MergeFrom(normalizer_spec)
            sp_model = model.SerializeToString()
            tokenizer.LoadFromSerializedProto(sp_model)
        return tokenizer

    @staticmethod
    def _eventually_correct_t5_max_length(pretrained_model_name_or_path, max_model_length, init_max_model_length):
        if pretrained_model_name_or_path in T5Tokenizer.max_model_input_sizes:
            deprecated_max_model_length = T5Tokenizer.max_model_input_sizes[pretrained_model_name_or_path]
            if init_max_model_length is not None and init_max_model_length != max_model_length:
                return init_max_model_length
            elif init_max_model_length is None:
                warnings.warn(
                    "This tokenizer was incorrectly instantiated with a model max length of"
                    f" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
                    " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
                    " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
                    f" {pretrained_model_name_or_path} automatically truncating your input to"
                    f" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
                    f" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
                    " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
                    " instantiate this tokenizer with `model_max_length` set to your preferred value.",
                    FutureWarning,
                )

        return max_model_length

    @property
    def vocab_size(self):
        return self.sp_model.get_piece_size()

    def get_vocab(self):
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        
        if token_ids_1 is None:
            return ([0] * len(token_ids_0)) + [1]
        return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]

    def get_sentinel_tokens(self):
        return list(
            set(filter(lambda x: bool(re.search(r"<extra_id_\d+>", x)) is not None, self.additional_special_tokens))
        )

    def get_sentinel_token_ids(self):
        return [self.convert_tokens_to_ids(token) for token in self.get_sentinel_tokens()]

    def _add_eos_if_not_present(self, token_ids: List[int]) -> List[int]:
        
        if len(token_ids) > 0 and token_ids[-1] == self.eos_token_id:
            warnings.warn(
                f"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"
                " eos tokens being added."
            )
            return token_ids
        else:
            return token_ids + [self.eos_token_id]

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        eos = [self.eos_token_id]

        if token_ids_1 is None:
            return len(token_ids_0 + eos) * [0]
        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        token_ids_0 = self._add_eos_if_not_present(token_ids_0)
        if token_ids_1 is None:
            return token_ids_0
        else:
            token_ids_1 = self._add_eos_if_not_present(token_ids_1)
            return token_ids_0 + token_ids_1

    def __getstate__(self):
        state = self.__dict__.copy()
        state["sp_model"] = None
        return state

    def __setstate__(self, d):
        self.__dict__ = d

        
        if not hasattr(self, "sp_model_kwargs"):
            self.sp_model_kwargs = {}

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(self.vocab_file)

    
    def tokenize(self, text: "TextInput", add_special_tokens=False, **kwargs) -> List[str]:
        
        if self.legacy or len(text) == 0:
            return super().tokenize(text, **kwargs)

        tokens = super().tokenize(SPIECE_UNDERLINE + text.replace(SPIECE_UNDERLINE, " "), **kwargs)

        if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:
            tokens = tokens[1:]
        return tokens

    @property
    def unk_token_length(self):
        return len(self.sp_model.encode(str(self.unk_token)))

    def _tokenize(self, text, **kwargs):
        
        tokens = self.sp_model.encode(text, out_type=str)
        if self.legacy or not text.startswith((SPIECE_UNDERLINE, " ")):
            return tokens

        
        tokens = self.sp_model.encode(self.unk_token + text, out_type=str)
        
        return tokens[self.unk_token_length :] if len(tokens) >= self.unk_token_length else tokens

    def _convert_token_to_id(self, token):
        
        return self.sp_model.piece_to_id(token)

    def _convert_id_to_token(self, index):
        
        token = self.sp_model.IdToPiece(index)
        return token

    def convert_tokens_to_string(self, tokens):
        
        current_sub_tokens = []
        
        tokens[0] = tokens[0].lstrip(SPIECE_UNDERLINE)
        out_string = ""
        prev_is_special = False
        for token in tokens:
            
            if token in self.all_special_tokens:
                if not prev_is_special:
                    out_string += " "
                out_string += self.sp_model.decode(current_sub_tokens) + token
                prev_is_special = True
                current_sub_tokens = []
            else:
                current_sub_tokens.append(token)
                prev_is_special = False
        out_string += self.sp_model.decode(current_sub_tokens)
        return out_string.strip()

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        return (out_vocab_file,)