codingutf8 2018 t5 s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model t5 import os import re import warnings from shutil import copyfile from typing import list optional tuple from tokenizationutilsfast import pretrainedtokenizerfast from utils import issentencepieceavailable logging if issentencepieceavailable from tokenizationt5 import t5tokenizer else t5tokenizer none logger logging getloggername vocabfilesnames vocabfile spiece model tokenizerfile tokenizer json pretrainedvocabfilesmap vocabfile t5small https huggingface cot5smallresolvemainspiece model t5base https huggingface cot5baseresolvemainspiece model t5large https huggingface cot5largeresolvemainspiece model t53b https huggingface cot53bresolvemainspiece model t511b https huggingface cot511bresolvemainspiece model tokenizerfile t5small https huggingface cot5smallresolvemaintokenizer json t5base https huggingface cot5baseresolvemaintokenizer json t5large https huggingface cot5largeresolvemaintokenizer json t53b https huggingface cot53bresolvemaintokenizer json t511b https huggingface cot511bresolvemaintokenizer json todopvp this should be removed in transformers v5 pretrainedpositionalembeddingssizes t5small 512 t5base 512 t5large 512 t53b 512 t511b 512 class t5tokenizerfastpretrainedtokenizerfast vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask slowtokenizerclass t5tokenizer prefixtokens listint def init self vocabfilenone tokenizerfilenone eostokens unktokenunk padtokenpad extraids100 additionalspecialtokensnone kwargs add extraids to the special token list if additionalspecialtokens is not none extratokens x for x in additionalspecialtokens if extraid in strx if lenextratokens 1 additionalspecialtokens fextraidi for i in rangeextraids elif extraids 0 and extraids lenextratokens raise valueerror fboth extraids extraids and additionalspecialtokens additionalspecialtokens are provided to t5tokenizer in this case the additionalspecialtokens must include the extraids tokens else extratokens fextraidi for i in rangeextraids additionalspecialtokens extratokens super init vocabfile tokenizerfiletokenizerfile eostokeneostoken unktokenunktoken padtokenpadtoken extraidsextraids additionalspecialtokensadditionalspecialtokens kwargs self vocabfile vocabfile self extraids extraids property def cansaveslowtokenizerself bool return os path isfileself vocabfile if self vocabfile else false staticmethod def eventuallycorrectt5maxlengthpretrainedmodelnameorpath maxmodellength initmaxmodellength if pretrainedmodelnameorpath in t5tokenizerfast maxmodelinputsizes deprecatedmaxmodellength t5tokenizerfast maxmodelinputsizespretrainedmodelnameorpath if initmaxmodellength is not none and initmaxmodellength maxmodellength return initmaxmodellength elif initmaxmodellength is none warnings warn this tokenizer was incorrectly instantiated with a model max length of f deprecatedmaxmodellength which will be corrected in transformers v5 nfor now this behavior is kept to avoid breaking backwards compatibility when paddingencoding with truncation is true n be aware that you should not rely on f pretrainedmodelnameorpath automatically truncating your input to f deprecatedmaxmodellength when paddingencoding n if you want to encodepad to sequences f longer than deprecatedmaxmodellength you can either instantiate this tokenizer with modelmaxlength or pass maxlength when encodingpadding n to avoid this warning please instantiate this tokenizer with modelmaxlength set to your preferred value futurewarning return maxmodellength def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr if not self cansaveslowtokenizer raise valueerror your fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer if not os path isdirsavedirectory logger errorfvocabulary path savedirectory should be a directory return outvocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesvocabfile if os path abspathself vocabfile os path abspathoutvocabfile copyfileself vocabfile outvocabfile logger infofcopy vocab file to outvocabfile return outvocabfile def buildinputswithspecialtokens self tokenids0 listint tokenids1 optionallistint none listint tokenids0 tokenids0 self eostokenid if tokenids1 is none return self prefixtokens tokenids0 else tokenids1 tokenids1 self eostokenid return self prefixtokens tokenids0 tokenids1 def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint eos self eostokenid if tokenids1 is none return lentokenids0 eos 0 return lentokenids0 eos tokenids1 eos 0 def getsentineltokensself return list setfilterlambda x boolre searchrextraidd x is not none self additionalspecialtokens def getsentineltokenidsself return self converttokenstoidstoken for token in self getsentineltokens coding utf 8 2018 t5 s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model t5 todo pvp this should be removed in transformers v5 construct a fast t5 tokenizer backed by huggingface s tokenizers library based on unigram https huggingface co docs tokenizers python latest components html highlight unigram models this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str sentencepiece https github com google sentencepiece file generally has a spm extension that contains the vocabulary necessary to instantiate a tokenizer eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths extra_ids int optional defaults to 100 add a number of extra ids added to the vocabulary for use as sentinels these tokens are accessible as extra_id_ d where d is a number between 0 and extra_ids 1 these tokens can be retrieved by calling get_sentinel_tokens method and token ids can be by calling get_sentinel_token_ids method additional_special_tokens list str optional additional special tokens used by the tokenizer add extra_ids to the special token list build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a sequence has the following format single sequence x s pair of sequences a s b s args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens create a mask from the two sequences passed to be used in a sequence pair classification task t5 does not make use of token type ids therefore a list of zeros is returned args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of zeros
import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_t5 import T5Tokenizer
else:
    T5Tokenizer = None


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/spiece.model",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/spiece.model",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/spiece.model",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/tokenizer.json",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/tokenizer.json",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/tokenizer.json",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/tokenizer.json",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/tokenizer.json",
    },
}



PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "t5-small": 512,
    "t5-base": 512,
    "t5-large": 512,
    "t5-3b": 512,
    "t5-11b": 512,
}


class T5TokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = T5Tokenizer

    prefix_tokens: List[int] = []

    def __init__(
        self,
        vocab_file=None,
        tokenizer_file=None,
        eos_token="</s>",
        unk_token="<unk>",
        pad_token="<pad>",
        extra_ids=100,
        additional_special_tokens=None,
        **kwargs,
    ):
        
        if additional_special_tokens is not None:
            extra_tokens = [x for x in additional_special_tokens if "<extra_id_" in str(x)]
            if len(extra_tokens) < 1:
                additional_special_tokens += [f"<extra_id_{i}>" for i in range(extra_ids)]
            elif extra_ids > 0 and extra_ids != len(extra_tokens):
                raise ValueError(
                    f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
                    " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
                    " tokens"
                )
        else:
            extra_tokens = [f"<extra_id_{i}>" for i in range(extra_ids)]
            additional_special_tokens = extra_tokens

        super().__init__(
            vocab_file,
            tokenizer_file=tokenizer_file,
            eos_token=eos_token,
            unk_token=unk_token,
            pad_token=pad_token,
            extra_ids=extra_ids,
            additional_special_tokens=additional_special_tokens,
            **kwargs,
        )

        self.vocab_file = vocab_file
        self._extra_ids = extra_ids

    @property
    def can_save_slow_tokenizer(self) -> bool:
        return os.path.isfile(self.vocab_file) if self.vocab_file else False

    @staticmethod
    def _eventually_correct_t5_max_length(pretrained_model_name_or_path, max_model_length, init_max_model_length):
        if pretrained_model_name_or_path in T5TokenizerFast.max_model_input_sizes:
            deprecated_max_model_length = T5TokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
            if init_max_model_length is not None and init_max_model_length != max_model_length:
                return init_max_model_length
            elif init_max_model_length is None:
                warnings.warn(
                    "This tokenizer was incorrectly instantiated with a model max length of"
                    f" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
                    " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
                    " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
                    f" {pretrained_model_name_or_path} automatically truncating your input to"
                    f" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
                    f" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
                    " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
                    " instantiate this tokenizer with `model_max_length` set to your preferred value.",
                    FutureWarning,
                )

        return max_model_length

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not self.can_save_slow_tokenizer:
            raise ValueError(
                "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                "tokenizer."
            )

        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
            logger.info(f"Copy vocab file to {out_vocab_file}")

        return (out_vocab_file,)

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        token_ids_0 = token_ids_0 + [self.eos_token_id]
        if token_ids_1 is None:
            return self.prefix_tokens + token_ids_0
        else:
            token_ids_1 = token_ids_1 + [self.eos_token_id]
            return self.prefix_tokens + token_ids_0 + token_ids_1

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        eos = [self.eos_token_id]

        if token_ids_1 is None:
            return len(token_ids_0 + eos) * [0]
        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

    def get_sentinel_tokens(self):
        return list(
            set(filter(lambda x: bool(re.search(r"<extra_id_\d+>", x)) is not None, self.additional_special_tokens))
        )

    def get_sentinel_token_ids(self):
        return [self.convert_tokens_to_ids(token) for token in self.get_sentinel_tokens()]