codingutf8 2020 microsoft and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license fast tokenization class for model deberta import json from typing import list optional tuple from tokenizers import pretokenizers from tokenizationutilsbase import addedtoken batchencoding from tokenizationutilsfast import pretrainedtokenizerfast from utils import logging from tokenizationdeberta import debertatokenizer logger logging getloggername vocabfilesnames vocabfile vocab json mergesfile merges txt tokenizerfile tokenizer json pretrainedvocabfilesmap vocabfile microsoftdebertabase https huggingface comicrosoftdebertabaseresolvemainvocab json microsoftdebertalarge https huggingface comicrosoftdebertalargeresolvemainvocab json microsoftdebertaxlarge https huggingface comicrosoftdebertaxlargeresolvemainvocab json microsoftdebertabasemnli https huggingface comicrosoftdebertabasemnliresolvemainvocab json microsoftdebertalargemnli https huggingface comicrosoftdebertalargemnliresolvemainvocab json microsoftdebertaxlargemnli https huggingface comicrosoftdebertaxlargemnliresolvemainvocab json mergesfile microsoftdebertabase https huggingface comicrosoftdebertabaseresolvemainmerges txt microsoftdebertalarge https huggingface comicrosoftdebertalargeresolvemainmerges txt microsoftdebertaxlarge https huggingface comicrosoftdebertaxlargeresolvemainmerges txt microsoftdebertabasemnli https huggingface comicrosoftdebertabasemnliresolvemainmerges txt microsoftdebertalargemnli https huggingface comicrosoftdebertalargemnliresolvemainmerges txt microsoftdebertaxlargemnli https huggingface comicrosoftdebertaxlargemnliresolvemainmerges txt pretrainedpositionalembeddingssizes microsoftdebertabase 512 microsoftdebertalarge 512 microsoftdebertaxlarge 512 microsoftdebertabasemnli 512 microsoftdebertalargemnli 512 microsoftdebertaxlargemnli 512 pretrainedinitconfiguration microsoftdebertabase dolowercase false microsoftdebertalarge dolowercase false class debertatokenizerfastpretrainedtokenizerfast vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask tokentypeids slowtokenizerclass debertatokenizer def init self vocabfilenone mergesfilenone tokenizerfilenone errorsreplace bostokencls eostokensep septokensep clstokencls unktokenunk padtokenpad masktokenmask addprefixspacefalse kwargs super init vocabfile mergesfile tokenizerfiletokenizerfile errorserrors bostokenbostoken eostokeneostoken unktokenunktoken septokenseptoken clstokenclstoken padtokenpadtoken masktokenmasktoken addprefixspaceaddprefixspace kwargs self addbostoken kwargs popaddbostoken false pretokstate json loadsself backendtokenizer pretokenizer getstate if pretokstate getaddprefixspace addprefixspace addprefixspace pretokclass getattrpretokenizers pretokstate poptype pretokstateaddprefixspace addprefixspace self backendtokenizer pretokenizer pretokclasspretokstate self addprefixspace addprefixspace property def masktokenself str if self masktoken is none if self verbose logger errorusing masktoken but it is not set yet return none return strself masktoken masktoken setter def masktokenself value mask token behave like a normal word i e include the space before it so we set lstrip to true value addedtokenvalue lstriptrue rstripfalse if isinstancevalue str else value self masktoken value def buildinputswithspecialtokens self tokenids0 listint tokenids1 optionallistint none listint if tokenids1 is none return self clstokenid tokenids0 self septokenid cls self clstokenid sep self septokenid return cls tokenids0 sep tokenids1 sep def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint sep self septokenid cls self clstokenid if tokenids1 is none return lencls tokenids0 sep 0 return lencls tokenids0 sep 0 lentokenids1 sep 1 copied from transformers models gpt2 tokenizationgpt2fast gpt2tokenizerfast batchencodeplus def batchencodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false assert self addprefixspace or not issplitintowords fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super batchencodeplusargs kwargs copied from transformers models gpt2 tokenizationgpt2fast gpt2tokenizerfast encodeplus def encodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false assert self addprefixspace or not issplitintowords fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super encodeplusargs kwargs copied from transformers models gpt2 tokenizationgpt2fast gpt2tokenizerfast savevocabulary def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles coding utf 8 2020 microsoft and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license fast tokenization class for model deberta construct a fast deberta tokenizer backed by huggingface s tokenizers library based on byte level byte pair encoding this tokenizer has been trained to treat spaces like parts of the tokens a bit like sentencepiece so a word will be encoded differently whether it is at the beginning of the sentence without space or not python from transformers import debertatokenizerfast tokenizer debertatokenizerfast from_pretrained microsoft deberta base tokenizer hello world input_ids 1 31414 232 2 tokenizer hello world input_ids 1 20920 232 2 you can get around that behavior by passing add_prefix_space true when instantiating this tokenizer but since the model was not pretrained this way it might yield a decrease in performance tip when used with is_split_into_words true this tokenizer needs to be instantiated with add_prefix_space true tip this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str optional path to the vocabulary file merges_file str optional path to the merges file tokenizer_file str optional the path to a tokenizer file to use instead of the vocab file errors str optional defaults to replace paradigm to follow when decoding bytes to utf 8 see bytes decode https docs python org 3 library stdtypes html bytes decode for more information bos_token str optional defaults to cls the beginning of sequence token eos_token str optional defaults to sep the end of sequence token sep_token str optional defaults to sep the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens cls_token str optional defaults to cls the classifier token which is used when doing sequence classification classification of the whole sequence instead of per token classification it is the first token of the sequence when built with special tokens unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths mask_token str optional defaults to mask the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict add_prefix_space bool optional defaults to false whether or not to add an initial space to the input this allows to treat the leading word just as any other word deberta tokenizer detect beginning of words by the preceding space str mask token to use when training a model with masked language modeling log an error if used while not having been set deberta tokenizer has a special mask token to be used in the fill mask pipeline the mask token will greedily comprise the space before the mask overriding the default behavior of the mask token to have it eat the space before it mask token behave like a normal word i e include the space before it so we set lstrip to true build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a deberta sequence has the following format single sequence cls x sep pair of sequences cls a sep b sep args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens create a mask from the two sequences passed to be used in a sequence pair classification task a deberta sequence pair mask has the following format 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 first sequence second sequence if token_ids_1 is none this method only returns the first portion of the mask 0s args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of token type ids glossary token type ids according to the given sequence s copied from transformers models gpt2 tokenization_gpt2_fast gpt2tokenizerfast _batch_encode_plus copied from transformers models gpt2 tokenization_gpt2_fast gpt2tokenizerfast _encode_plus copied from transformers models gpt2 tokenization_gpt2_fast gpt2tokenizerfast save_vocabulary
import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"
        ),
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "microsoft/deberta-base": 512,
    "microsoft/deberta-large": 512,
    "microsoft/deberta-xlarge": 512,
    "microsoft/deberta-base-mnli": 512,
    "microsoft/deberta-large-mnli": 512,
    "microsoft/deberta-xlarge-mnli": 512,
}

PRETRAINED_INIT_CONFIGURATION = {
    "microsoft/deberta-base": {"do_lower_case": False},
    "microsoft/deberta-large": {"do_lower_case": False},
}


class DebertaTokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask", "token_type_ids"]
    slow_tokenizer_class = DebertaTokenizer

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        errors="replace",
        bos_token="[CLS]",
        eos_token="[SEP]",
        sep_token="[SEP]",
        cls_token="[CLS]",
        unk_token="[UNK]",
        pad_token="[PAD]",
        mask_token="[MASK]",
        add_prefix_space=False,
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            errors=errors,
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            sep_token=sep_token,
            cls_token=cls_token,
            pad_token=pad_token,
            mask_token=mask_token,
            add_prefix_space=add_prefix_space,
            **kwargs,
        )
        self.add_bos_token = kwargs.pop("add_bos_token", False)

        pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
        if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
            pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
            pre_tok_state["add_prefix_space"] = add_prefix_space
            self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

        self.add_prefix_space = add_prefix_space

    @property
    def mask_token(self) -> str:
        
        if self._mask_token is None:
            if self.verbose:
                logger.error("Using mask_token, but it is not set yet.")
            return None
        return str(self._mask_token)

    @mask_token.setter
    def mask_token(self, value):
        
        
        
        value = AddedToken(value, lstrip=True, rstrip=False) if isinstance(value, str) else value
        self._mask_token = value

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        if token_ids_1 is None:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        return cls + token_ids_0 + sep + token_ids_1 + sep

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

    
    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)
        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*args, **kwargs)

    
    def _encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)

        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*args, **kwargs)

    
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)