codingutf8 2021 google research and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for bigbird import os import re from shutil import copyfile from typing import any dict list optional tuple import sentencepiece as spm from tokenizationutils import addedtoken pretrainedtokenizer from utils import logging logger logging getloggername vocabfilesnames vocabfile spiece model pretrainedvocabfilesmap vocabfile googlebigbirdrobertabase https huggingface cogooglebigbirdrobertabaseresolvemainspiece model googlebigbirdrobertalarge https huggingface cogooglebigbirdrobertalargeresolvemainspiece model googlebigbirdbasetriviaitc https huggingface cogooglebigbirdbasetriviaitcresolvemainspiece model pretrainedpositionalembeddingssizes googlebigbirdrobertabase 4096 googlebigbirdrobertalarge 4096 googlebigbirdbasetriviaitc 4096 class bigbirdtokenizerpretrainedtokenizer vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask prefixtokens listint def init self vocabfile unktokenunk bostokens eostokens padtokenpad septokensep masktokenmask clstokencls spmodelkwargs optionaldictstr any none kwargs none bostoken addedtokenbostoken lstripfalse rstripfalse if isinstancebostoken str else bostoken eostoken addedtokeneostoken lstripfalse rstripfalse if isinstanceeostoken str else eostoken unktoken addedtokenunktoken lstripfalse rstripfalse if isinstanceunktoken str else unktoken padtoken addedtokenpadtoken lstripfalse rstripfalse if isinstancepadtoken str else padtoken clstoken addedtokenclstoken lstripfalse rstripfalse if isinstanceclstoken str else clstoken septoken addedtokenseptoken lstripfalse rstripfalse if isinstanceseptoken str else septoken mask token behave like a normal word i e include the space before it masktoken addedtokenmasktoken lstriptrue rstripfalse if isinstancemasktoken str else masktoken self spmodelkwargs if spmodelkwargs is none else spmodelkwargs self vocabfile vocabfile self spmodel spm sentencepieceprocessorself spmodelkwargs self spmodel loadvocabfile super init bostokenbostoken eostokeneostoken unktokenunktoken padtokenpadtoken septokenseptoken masktokenmasktoken clstokenclstoken spmodelkwargsself spmodelkwargs kwargs property def vocabsizeself return self spmodel getpiecesize def getvocabself vocab self convertidstotokensi i for i in rangeself vocabsize vocab updateself addedtokensencoder return vocab def getstateself state self dict copy statespmodel none return state def setstateself d self dict d for backward compatibility if not hasattrself spmodelkwargs self spmodelkwargs self spmodel spm sentencepieceprocessorself spmodelkwargs self spmodel loadself vocabfile def tokenizeself text str liststr converts a token str in an id using the vocab return self spmodel piecetoidtoken def convertidtotokenself index converts a sequence of tokens string in a single string currentsubtokens outstring previsspecial false for token in tokens make sure that special tokens are not decoded using sentencepiece model if token in self allspecialtokens if not previsspecial outstring outstring self spmodel decodecurrentsubtokens token previsspecial true currentsubtokens else currentsubtokens appendtoken previsspecial false outstring self spmodel decodecurrentsubtokens return outstring strip def decode self tokenids listint skipspecialtokens bool false cleanuptokenizationspaces bool none spacesbetweenspecialtokens bool true kwargs str self decodeusesourcetokenizer kwargs popusesourcetokenizer false filteredtokens self convertidstotokenstokenids skipspecialtokensskipspecialtokens to avoid mixing bytelevel and unicode for bytelevel bpt we need to build string separately for added tokens and bytelevel tokens cf https github comhuggingfacetransformersissues1133 subtexts currentsubtext for token in filteredtokens if skipspecialtokens and token in self allspecialids continue if token in self addedtokensencoder if currentsubtext subtexts appendself converttokenstostringcurrentsubtext currentsubtext subtexts appendtoken else currentsubtext appendtoken if currentsubtext subtexts appendself converttokenstostringcurrentsubtext mimic the behavior of the rust tokenizer no space before mask and sep if spacesbetweenspecialtokens text re subr masksep r1 joinsubtexts else text joinsubtexts cleanuptokenizationspaces cleanuptokenizationspaces if cleanuptokenizationspaces is not none else self cleanuptokenizationspaces if cleanuptokenizationspaces cleantext self cleanuptokenizationtext return cleantext else return text def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr if not os path isdirsavedirectory logger errorfvocabulary path savedirectory should be a directory return outvocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesvocabfile if os path abspathself vocabfile os path abspathoutvocabfile and os path isfileself vocabfile copyfileself vocabfile outvocabfile elif not os path isfileself vocabfile with openoutvocabfile wb as fi contentspiecemodel self spmodel serializedmodelproto fi writecontentspiecemodel return outvocabfile def buildinputswithspecialtokens self tokenids0 listint tokenids1 optionallistint none listint if tokenids1 is none return self clstokenid tokenids0 self septokenid cls self clstokenid sep self septokenid return cls tokenids0 sep tokenids1 sep def getspecialtokensmask self tokenids0 listint tokenids1 optionallistint none alreadyhasspecialtokens bool false listint if alreadyhasspecialtokens return super getspecialtokensmask tokenids0tokenids0 tokenids1tokenids1 alreadyhasspecialtokenstrue if tokenids1 is none return 1 0 lentokenids0 1 return 1 0 lentokenids0 1 0 lentokenids1 1 def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint sep self septokenid cls self clstokenid if tokenids1 is none return lencls tokenids0 sep 0 return lencls tokenids0 sep 0 lentokenids1 sep 1 coding utf 8 2021 google research and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for bigbird construct a bigbird tokenizer based on sentencepiece https github com google sentencepiece this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str sentencepiece https github com google sentencepiece file generally has a spm extension that contains the vocabulary necessary to instantiate a tokenizer unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead bos_token str optional defaults to s the begin of sequence token eos_token str optional defaults to s the end of sequence token pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths sep_token str optional defaults to sep the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens mask_token str optional defaults to mask the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict cls_token str optional defaults to cls the classifier token which is used when doing sequence classification classification of the whole sequence instead of per token classification it is the first token of the sequence when built with special tokens sp_model_kwargs dict optional will be passed to the sentencepieceprocessor __init__ method the python wrapper for sentencepiece https github com google sentencepiece tree master python can be used among other things to set enable_sampling enable subword regularization nbest_size sampling parameters for unigram invalid for bpe dropout nbest_size 0 1 no sampling is performed nbest_size 1 samples from the nbest_size results nbest_size 0 assuming that nbest_size is infinite and samples from the all hypothesis lattice using forward filtering and backward sampling algorithm alpha smoothing parameter for unigram sampling and dropout probability of merge operations for bpe dropout mask token behave like a normal word i e include the space before it for backward compatibility take as input a string and return a list of strings tokens for words sub words converts a token str in an id using the vocab converts an index integer in a token str using the vocab converts a sequence of tokens string in a single string make sure that special tokens are not decoded using sentencepiece model to avoid mixing byte level and unicode for byte level bpt we need to build string separately for added tokens and byte level tokens cf https github com huggingface transformers issues 1133 mimic the behavior of the rust tokenizer no space before mask and sep build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a big bird sequence has the following format single sequence cls x sep pair of sequences cls a sep b sep args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepare_for_model method args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs already_has_special_tokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns list int a list of integers in the range 0 1 1 for a special token 0 for a sequence token create a mask from the two sequences passed to be used in a sequence pair classification task a bert sequence pair mask has the following format 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 first sequence second sequence if token_ids_1 is none this method only returns the first portion of the mask 0s args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of token type ids glossary token type ids according to the given sequence s
import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "google/bigbird-roberta-base": "https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model",
        "google/bigbird-roberta-large": (
            "https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"
        ),
        "google/bigbird-base-trivia-itc": (
            "https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"
        ),
    }
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "google/bigbird-roberta-base": 4096,
    "google/bigbird-roberta-large": 4096,
    "google/bigbird-base-trivia-itc": 4096,
}


class BigBirdTokenizer(PreTrainedTokenizer):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    prefix_tokens: List[int] = []

    def __init__(
        self,
        vocab_file,
        unk_token="<unk>",
        bos_token="<s>",
        eos_token="</s>",
        pad_token="<pad>",
        sep_token="[SEP]",
        mask_token="[MASK]",
        cls_token="[CLS]",
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        **kwargs,
    ) -> None:
        bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
        eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
        unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
        cls_token = AddedToken(cls_token, lstrip=False, rstrip=False) if isinstance(cls_token, str) else cls_token
        sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token

        
        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token

        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs

        self.vocab_file = vocab_file

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(vocab_file)

        super().__init__(
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            pad_token=pad_token,
            sep_token=sep_token,
            mask_token=mask_token,
            cls_token=cls_token,
            sp_model_kwargs=self.sp_model_kwargs,
            **kwargs,
        )

    @property
    def vocab_size(self):
        return self.sp_model.get_piece_size()

    def get_vocab(self):
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    def __getstate__(self):
        state = self.__dict__.copy()
        state["sp_model"] = None
        return state

    def __setstate__(self, d):
        self.__dict__ = d

        
        if not hasattr(self, "sp_model_kwargs"):
            self.sp_model_kwargs = {}

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(self.vocab_file)

    def _tokenize(self, text: str) -> List[str]:
        
        return self.sp_model.encode(text, out_type=str)

    def _convert_token_to_id(self, token):
        
        return self.sp_model.piece_to_id(token)

    def _convert_id_to_token(self, index):
        
        token = self.sp_model.IdToPiece(index)
        return token

    def convert_tokens_to_string(self, tokens):
        
        current_sub_tokens = []
        out_string = ""
        prev_is_special = False
        for token in tokens:
            
            if token in self.all_special_tokens:
                if not prev_is_special:
                    out_string += " "
                out_string += self.sp_model.decode(current_sub_tokens) + token
                prev_is_special = True
                current_sub_tokens = []
            else:
                current_sub_tokens.append(token)
                prev_is_special = False
        out_string += self.sp_model.decode(current_sub_tokens)
        return out_string.strip()

    def _decode(
        self,
        token_ids: List[int],
        skip_special_tokens: bool = False,
        clean_up_tokenization_spaces: bool = None,
        spaces_between_special_tokens: bool = True,
        **kwargs,
    ) -> str:
        self._decode_use_source_tokenizer = kwargs.pop("use_source_tokenizer", False)

        filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)

        
        
        
        sub_texts = []
        current_sub_text = []
        for token in filtered_tokens:
            if skip_special_tokens and token in self.all_special_ids:
                continue
            if token in self.added_tokens_encoder:
                if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(current_sub_text))
                    current_sub_text = []
                sub_texts.append(token)
            else:
                current_sub_text.append(token)
        if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(current_sub_text))

        
        
        if spaces_between_special_tokens:
            text = re.sub(r" (\[(MASK|SEP)\])", r"\1", " ".join(sub_texts))
        else:
            text = "".join(sub_texts)

        clean_up_tokenization_spaces = (
            clean_up_tokenization_spaces
            if clean_up_tokenization_spaces is not None
            else self.clean_up_tokenization_spaces
        )
        if clean_up_tokenization_spaces:
            clean_text = self.clean_up_tokenization(text)
            return clean_text
        else:
            return text

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        return (out_vocab_file,)

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        if token_ids_1 is None:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        return cls + token_ids_0 + sep + token_ids_1 + sep

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        if token_ids_1 is None:
            return [1] + ([0] * len(token_ids_0)) + [1]
        return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]