codingutf8 2021 t5 s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model byt5 import warnings from typing import list optional tuple from tokenizationutils import addedtoken pretrainedtokenizer from utils import logging logger logging getloggername class byt5tokenizerpretrainedtokenizer modelinputnames inputids attentionmask def init self eostokens unktokenunk padtokenpad extraids125 additionalspecialtokensnone kwargs none add extraids to the special token list if extraids 0 and additionalspecialtokens is none additionalspecialtokens fextraidi for i in rangeextraids elif extraids 0 and additionalspecialtokens is not none and lenadditionalspecialtokens 0 check that we have the right number of extraid special tokens extratokens lensetfilterlambda x boolextraid in strx additionalspecialtokens if extratokens extraids raise valueerror fboth extraids extraids and additionalspecialtokens additionalspecialtokens are provided to byt5tokenizer in this case the additionalspecialtokens must include the extraids tokens padtoken addedtokenpadtoken lstriptrue rstriptrue if isinstancepadtoken str else padtoken we force left and right stripping for backward compatibility the byt5tests depend on this eostoken addedtokeneostoken lstriptrue rstriptrue if isinstanceeostoken str else eostoken unktoken addedtokenunktoken lstriptrue rstriptrue if isinstanceunktoken str else unktoken unk token needs to be in the vocab with correct index self addedtokensdecoder 0 padtoken 1 eostoken 2 unktoken self offset lenself addedtokensdecoder self utfvocabsize 28 utf is 8 bits super init eostokeneostoken unktokenunktoken padtokenpadtoken extraids0 additionalspecialtokensadditionalspecialtokens todo extra ids are not used sweatywmile kwargs property def vocabsizeself return self utfvocabsize def getvocabself vocab self convertidstotokensi i for i in rangeself vocabsize self offset vocab updateself addedtokensencoder return vocab def getspecialtokensmask self tokenids0 listint tokenids1 optionallistint none alreadyhasspecialtokens bool false listint if alreadyhasspecialtokens return super getspecialtokensmask tokenids0tokenids0 tokenids1tokenids1 alreadyhasspecialtokenstrue normal case some special tokens if tokenids1 is none return 0 lentokenids0 1 return 0 lentokenids0 1 0 lentokenids1 1 def addeosifnotpresentself tokenids listint listint create a mask from the two sequences passed to be used in a sequencepair classification task byt5 does not make use of token type ids therefore a list of zeros is returned args tokenids0 listint list of ids tokenids1 listint optional optional second list of ids for sequence pairs returns listint list of zeros build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a sequence has the following format single sequence x s pair of sequences a s b s args tokenids0 listint list of ids to which the special tokens will be added tokenids1 listint optional optional second list of ids for sequence pairs returns listint list of input ids glossaryinputids with the appropriate special tokens take as input a string and return a list of strings tokens for wordssubwords tokens chri for i in text encodeutf8 return tokens def converttokentoidself token converts an index integer in a token str using the vocab token chrindex self offset return token def converttokenstostringself tokens byt5tokenizer has no vocab file coding utf 8 2021 t5 s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model byt5 construct a byt5 tokenizer byt5 simply uses raw bytes utf 8 encoding this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths extra_ids int optional defaults to 125 add a number of extra ids added to the end of the vocabulary for use as sentinels these tokens are accessible as extra_id_ d where d is a number between 0 and extra_ids 1 extra tokens are indexed from the end of the vocabulary up to beginning extra_id_0 is the last token in the vocabulary like in byt5 preprocessing see here https github com google research text to text transfer transformer blob 9fd7b14a769417be33bc6c850f9598764913c833 t5 data preprocessors py l2117 additional_special_tokens list str optional additional special tokens used by the tokenizer add extra_ids to the special token list check that we have the right number of extra_id special tokens we force left and right stripping for backward compatibility the byt5tests depend on this unk token needs to be in the vocab with correct index utf is 8 bits todo extra ids are not used sweatywmile retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepare_for_model method args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs already_has_special_tokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns list int a list of integers in the range 0 1 1 for a special token 0 for a sequence token normal case some special tokens do not add eos again if user already added it create a mask from the two sequences passed to be used in a sequence pair classification task byt5 does not make use of token type ids therefore a list of zeros is returned args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of zeros build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a sequence has the following format single sequence x s pair of sequences a s b s args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens take as input a string and return a list of strings tokens for words sub words converts a token str in an id using the vocab converts an index integer in a token str using the vocab converts a sequence of tokens string in a single string byt5tokenizer has no vocab file
import warnings
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


logger = logging.get_logger(__name__)


class ByT5Tokenizer(PreTrainedTokenizer):
    

    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        eos_token="</s>",
        unk_token="<unk>",
        pad_token="<pad>",
        extra_ids=125,
        additional_special_tokens=None,
        **kwargs,
    ) -> None:
        
        if extra_ids > 0 and additional_special_tokens is None:
            additional_special_tokens = [f"<extra_id_{i}>" for i in range(extra_ids)]
        elif extra_ids > 0 and additional_special_tokens is not None and len(additional_special_tokens) > 0:
            
            extra_tokens = len(set(filter(lambda x: bool("extra_id" in str(x)), additional_special_tokens)))
            if extra_tokens != extra_ids:
                raise ValueError(
                    f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
                    " provided to ByT5Tokenizer. In this case the additional_special_tokens must include the"
                    " extra_ids tokens"
                )

        pad_token = AddedToken(pad_token, lstrip=True, rstrip=True) if isinstance(pad_token, str) else pad_token
        
        eos_token = AddedToken(eos_token, lstrip=True, rstrip=True) if isinstance(eos_token, str) else eos_token
        unk_token = AddedToken(unk_token, lstrip=True, rstrip=True) if isinstance(unk_token, str) else unk_token
        
        self._added_tokens_decoder = {0: pad_token, 1: eos_token, 2: unk_token}
        self.offset = len(self._added_tokens_decoder)
        self._utf_vocab_size = 2**8  
        super().__init__(
            eos_token=eos_token,
            unk_token=unk_token,
            pad_token=pad_token,
            extra_ids=0,
            additional_special_tokens=additional_special_tokens,  
            **kwargs,
        )

    @property
    def vocab_size(self):
        return self._utf_vocab_size

    def get_vocab(self):
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size + self.offset)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        
        if token_ids_1 is None:
            return ([0] * len(token_ids_0)) + [1]
        return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]

    def _add_eos_if_not_present(self, token_ids: List[int]) -> List[int]:
        
        if len(token_ids) > 0 and token_ids[-1] == self.eos_token_id:
            warnings.warn(
                f"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"
                " eos tokens being added."
            )
            return token_ids
        else:
            return token_ids + [self.eos_token_id]

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        eos = [self.eos_token_id]

        if token_ids_1 is None:
            return len(token_ids_0 + eos) * [0]
        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        token_ids_0 = self._add_eos_if_not_present(token_ids_0)
        if token_ids_1 is None:
            return token_ids_0
        else:
            token_ids_1 = self._add_eos_if_not_present(token_ids_1)
            return token_ids_0 + token_ids_1

    def _tokenize(self, text: str) -> List[str]:
        
        tokens = [chr(i) for i in text.encode("utf-8")]
        return tokens

    def _convert_token_to_id(self, token):
        

        if len(token) != 1:
            token_id = None
        else:
            token_id = ord(token) + self.offset

        return token_id

    def _convert_id_to_token(self, index):
        
        token = chr(index - self.offset)
        return token

    def convert_tokens_to_string(self, tokens):
        
        bstring = b""
        for token in tokens:
            if token in self.added_tokens_decoder:
                tok_string = self.added_tokens_decoder[token].encode("utf-8")
            elif token in self.added_tokens_encoder:
                tok_string = token.encode("utf-8")
            else:
                tok_string = bytes([ord(token)])
            bstring += tok_string
        string = bstring.decode("utf-8", errors="ignore")
        return string

    
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        return ()