codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for perceiver from typing import dict list optional tuple from tokenizationutils import addedtoken pretrainedtokenizer from utils import logging logger logging getloggername class perceivertokenizerpretrainedtokenizer modelinputnames inputids attentionmask def init self padtokenpad bostokenbos eostokeneos masktokenmask clstokencls septokensep modelmaxlength2048 kwargs none padtoken addedtokenpadtoken lstripfalse rstripfalse if isinstancepadtoken str else padtoken bostoken addedtokenbostoken lstripfalse rstripfalse if isinstancebostoken str else bostoken eostoken addedtokeneostoken lstripfalse rstripfalse if isinstanceeostoken str else eostoken masktoken addedtokenmasktoken lstripfalse rstripfalse if isinstancemasktoken str else masktoken clstoken addedtokenclstoken lstripfalse rstripfalse if isinstanceclstoken str else clstoken septoken addedtokenseptoken lstripfalse rstripfalse if isinstanceseptoken str else septoken self utfvocabsize 28 utf is 8 bits since these tokens are not part of the vocabulary we manually add them self addedtokensdecoder dictstr int 0 padtoken 1 bostoken 2 eostoken 3 masktoken 4 clstoken 5 septoken self numspecialtokens lenself addedtokensdecoder super init padtokenpadtoken bostokenbostoken eostokeneostoken masktokenmasktoken clstokenclstoken septokenseptoken modelmaxlengthmodelmaxlength kwargs def getvocabself dictstr int vocab for i in rangeself utfvocabsize token chri vocabtoken i self numspecialtokens vocab updateself addedtokensencoder return vocab property def vocabsizeself return self utfvocabsize def getspecialtokensmask self tokenids0 listint tokenids1 optionallistint none alreadyhasspecialtokens bool false listint if alreadyhasspecialtokens return super getspecialtokensmask tokenids0tokenids0 tokenids1tokenids1 alreadyhasspecialtokenstrue normal case some special tokens if tokenids1 is none return 1 0 lentokenids0 1 return 1 0 lentokenids0 1 0 lentokenids1 1 def buildinputswithspecialtokens self tokenids0 listint tokenids1 optionallistint none listint if tokenids1 is none return self clstokenid tokenids0 self septokenid else return self clstokenid tokenids0 self septokenid tokenids1 self septokenid def tokenizeself text str liststr converts a token str in an id using the vocab if lentoken 1 tokenid self unktokenid else tokenid ordtoken self numspecialtokens return tokenid def convertidtotokenself index todo arthurz refactor this as well converts a sequence of tokens string in a single string bstring b for token in tokens if token in self addedtokensencoder tokstring strtoken encodeutf8 else tokstring bytesordtoken bstring tokstring string bstring decodeutf8 errorsreplace return string perceivertokenizer has no vocab file def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr return coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for perceiver construct a perceiver tokenizer the perceiver simply uses raw bytes utf 8 encoding this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths bos_token str optional defaults to bos the bos token reserved in the vocab but not actually used eos_token str optional defaults to eos the end of sequence token reserved in the vocab but not actually used tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip mask_token str optional defaults to mask the mask token useful for masked language modeling cls_token str optional defaults to cls the cls token reserved in the vocab but not actually used sep_token str optional defaults to sep the separator token which is used when building a sequence from two sequences utf is 8 bits since these tokens are not part of the vocabulary we manually add them retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepare_for_model method args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs already_has_special_tokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns list int a list of integers in the range 0 1 1 for a special token 0 for a sequence token normal case some special tokens build model inputs from a sequence or a pair of sequence for sequence classification tasks a sequence has the following format single sequence cls x sep pair of sequences cls a sep b sep args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens take as input a string and return a list of strings tokens for words sub words converts a token str in an id using the vocab converts an index integer in a token str using the vocab todo arthurz refactor this as well converts a sequence of tokens string in a single string perceivertokenizer has no vocab file
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


logger = logging.get_logger(__name__)


class PerceiverTokenizer(PreTrainedTokenizer):
    

    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        pad_token="[PAD]",
        bos_token="[BOS]",
        eos_token="[EOS]",
        mask_token="[MASK]",
        cls_token="[CLS]",
        sep_token="[SEP]",
        model_max_length=2048,
        **kwargs,
    ) -> None:
        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
        bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
        eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
        mask_token = AddedToken(mask_token, lstrip=False, rstrip=False) if isinstance(mask_token, str) else mask_token
        cls_token = AddedToken(cls_token, lstrip=False, rstrip=False) if isinstance(cls_token, str) else cls_token
        sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token

        self._utf_vocab_size = 2**8  

        
        self._added_tokens_decoder: Dict[str, int] = {
            0: pad_token,
            1: bos_token,
            2: eos_token,
            3: mask_token,
            4: cls_token,
            5: sep_token,
        }
        self._num_special_tokens = len(self._added_tokens_decoder)
        super().__init__(
            pad_token=pad_token,
            bos_token=bos_token,
            eos_token=eos_token,
            mask_token=mask_token,
            cls_token=cls_token,
            sep_token=sep_token,
            model_max_length=model_max_length,
            **kwargs,
        )

    def get_vocab(self) -> Dict[str, int]:
        vocab = {}
        for i in range(self._utf_vocab_size):
            token = chr(i)
            vocab[token] = i + self._num_special_tokens
        vocab.update(self.added_tokens_encoder)
        return vocab

    @property
    def vocab_size(self):
        return self._utf_vocab_size

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        
        if token_ids_1 is None:
            return [1] + [0] * len(token_ids_0) + [1]
        return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        if token_ids_1 is None:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        else:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + token_ids_1 + [self.sep_token_id]

    def _tokenize(self, text: str) -> List[str]:
        
        tokens = [chr(i) for i in text.encode("utf-8")]
        return tokens

    def _convert_token_to_id(self, token):
        
        if len(token) != 1:
            token_id = self.unk_token_id
        else:
            token_id = ord(token) + self._num_special_tokens
        return token_id

    def _convert_id_to_token(self, index):
        
        token = chr(index - self._num_special_tokens)
        return token

    
    def convert_tokens_to_string(self, tokens):
        
        bstring = b""
        for token in tokens:
            if token in self.added_tokens_encoder:
                tok_string = str(token).encode("utf-8")
            else:
                tok_string = bytes([ord(token)])
            bstring += tok_string
        string = bstring.decode("utf-8", errors="replace")
        return string

    
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        return ()