codingutf8 2021 the facebook inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license fast tokenization class for blenderbot import json from typing import list optional tuple from tokenizers import pretokenizers processors from tokenizationutilsbase import addedtoken batchencoding from tokenizationutilsfast import pretrainedtokenizerfast from utils import logging from tokenizationblenderbot import blenderbottokenizer logger logging getloggername vocabfilesnames vocabfile vocab json mergesfile merges txt tokenizerconfigfile tokenizerconfig json pretrainedvocabfilesmap vocabfile facebookblenderbot3b https huggingface cofacebookblenderbot3bresolvemainvocab json mergesfile facebookblenderbot3b https huggingface cofacebookblenderbot3bresolvemainmerges txt tokenizerconfigfile facebookblenderbot3b https huggingface cofacebookblenderbot3bresolvemaintokenizerconfig json pretrainedpositionalembeddingssizes facebookblenderbot3b 128 class blenderbottokenizerfastpretrainedtokenizerfast vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask slowtokenizerclass blenderbottokenizer copied from transformers models roberta tokenizationrobertafast robertatokenizerfast init with robertablenderbot robertablenderbot def init self vocabfilenone mergesfilenone tokenizerfilenone errorsreplace bostokens eostokens septokens clstokens unktokenunk padtokenpad masktokenmask addprefixspacefalse trimoffsetstrue kwargs masktoken addedtokenmasktoken lstriptrue rstripfalse normalizedfalse if isinstancemasktoken str else masktoken super init vocabfile mergesfile tokenizerfiletokenizerfile errorserrors bostokenbostoken eostokeneostoken septokenseptoken clstokenclstoken unktokenunktoken padtokenpadtoken masktokenmasktoken addprefixspaceaddprefixspace trimoffsetstrimoffsets kwargs pretokstate json loadsself backendtokenizer pretokenizer getstate if pretokstate getaddprefixspace addprefixspace addprefixspace pretokclass getattrpretokenizers pretokstate poptype pretokstateaddprefixspace addprefixspace self backendtokenizer pretokenizer pretokclasspretokstate self addprefixspace addprefixspace tokenizercomponent postprocessor tokenizercomponentinstance getattrself backendtokenizer tokenizercomponent none if tokenizercomponentinstance state json loadstokenizercomponentinstance getstate the lists sep and cls must be cased in tuples for the object postprocessorclass if sep in state statesep tuplestatesep if cls in state statecls tuplestatecls changestoapply false if state getaddprefixspace addprefixspace addprefixspace stateaddprefixspace addprefixspace changestoapply true if state gettrimoffsets trimoffsets trimoffsets statetrimoffsets trimoffsets changestoapply true if changestoapply componentclass getattrprocessors state poptype newvalue componentclassstate setattrself backendtokenizer tokenizercomponent newvalue property copied from transformers models roberta tokenizationrobertafast robertatokenizerfast masktoken with robertablenderbot robertablenderbot def masktokenself str if self masktoken is none if self verbose logger errorusing masktoken but it is not set yet return none return strself masktoken masktoken setter def masktokenself value mask token behave like a normal word i e include the space before it so we set lstrip to true value addedtokenvalue lstriptrue rstripfalse if isinstancevalue str else value self masktoken value copied from transformers models roberta tokenizationrobertafast robertatokenizerfast batchencodeplus with robertablenderbot robertablenderbot def batchencodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false assert self addprefixspace or not issplitintowords fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super batchencodeplusargs kwargs copied from transformers models roberta tokenizationrobertafast robertatokenizerfast encodeplus with robertablenderbot robertablenderbot def encodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false assert self addprefixspace or not issplitintowords fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super encodeplusargs kwargs copied from transformers models roberta tokenizationrobertafast robertatokenizerfast savevocabulary with robertablenderbot robertablenderbot def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles copied from transformers models roberta tokenizationrobertafast robertatokenizerfast createtokentypeidsfromsequences with robertablenderbot robertablenderbot def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint sep self septokenid cls self clstokenid if tokenids1 is none return lencls tokenids0 sep 0 return lencls tokenids0 sep sep tokenids1 sep 0 def buildinputswithspecialtokensself tokenids0 listint tokenids1 optionallistint none return tokenids0 self eostokenid property copied from transformers models blenderbot tokenizationblenderbot blenderbottokenizer defaultchattemplate def defaultchattemplateself logger warningonce nno chat template is defined for this tokenizer using the default template ffor the self class name class if the default is not appropriate for your model please set tokenizer chattemplate to an appropriate template see https huggingface codocstransformersmainchattemplating for more information n return for message in messages if message role user endif message content if not loop last endif endfor eostoken coding utf 8 2021 the facebook inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license fast tokenization class for blenderbot construct a fast blenderbot tokenizer backed by huggingface s tokenizers library derived from the gpt 2 tokenizer using byte level byte pair encoding this tokenizer has been trained to treat spaces like parts of the tokens a bit like sentencepiece so a word will be encoded differently whether it is at the beginning of the sentence without space or not python from transformers import blenderbottokenizerfast tokenizer blenderbottokenizerfast from_pretrained facebook blenderbot 3b tokenizer hello world input_ids 6950 1085 2 tokenizer hello world input_ids 6950 1085 2 you can get around that behavior by passing add_prefix_space true when instantiating this tokenizer or when you call it on some text but since the model was not pretrained this way it might yield a decrease in performance tip when used with is_split_into_words true this tokenizer needs to be instantiated with add_prefix_space true tip this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str path to the vocabulary file merges_file str path to the merges file errors str optional defaults to replace paradigm to follow when decoding bytes to utf 8 see bytes decode https docs python org 3 library stdtypes html bytes decode for more information bos_token str optional defaults to s the beginning of sequence token that was used during pretraining can be used a sequence classifier token tip when building a sequence using special tokens this is not the token that is used for the beginning of sequence the token used is the cls_token tip eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip sep_token str optional defaults to s the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens cls_token str optional defaults to s the classifier token which is used when doing sequence classification classification of the whole sequence instead of per token classification it is the first token of the sequence when built with special tokens unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths mask_token str optional defaults to mask the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict add_prefix_space bool optional defaults to false whether or not to add an initial space to the input this allows to treat the leading word just as any other word blenderbot tokenizer detect beginning of words by the preceding space trim_offsets bool optional defaults to true whether the post processing step should trim offsets to avoid including whitespaces copied from transformers models roberta tokenization_roberta_fast robertatokenizerfast __init__ with roberta blenderbot roberta blenderbot the lists sep and cls must be cased in tuples for the object post_processor_class copied from transformers models roberta tokenization_roberta_fast robertatokenizerfast mask_token with roberta blenderbot roberta blenderbot str mask token to use when training a model with masked language modeling log an error if used while not having been set blenderbot tokenizer has a special mask token to be usable in the fill mask pipeline the mask token will greedily comprise the space before the mask overriding the default behavior of the mask token to have it eat the space before it this is needed to preserve backward compatibility with all the previously used models based on roberta mask token behave like a normal word i e include the space before it so we set lstrip to true copied from transformers models roberta tokenization_roberta_fast robertatokenizerfast _batch_encode_plus with roberta blenderbot roberta blenderbot copied from transformers models roberta tokenization_roberta_fast robertatokenizerfast _encode_plus with roberta blenderbot roberta blenderbot copied from transformers models roberta tokenization_roberta_fast robertatokenizerfast save_vocabulary with roberta blenderbot roberta blenderbot copied from transformers models roberta tokenization_roberta_fast robertatokenizerfast create_token_type_ids_from_sequences with roberta blenderbot roberta blenderbot create a mask from the two sequences passed to be used in a sequence pair classification task blenderbot does not make use of token type ids therefore a list of zeros is returned args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of zeros build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a blenderbot sequence has the following format single sequence x s args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional will be ignored returns list int list of input ids glossary input ids with the appropriate special tokens copied from transformers models blenderbot tokenization_blenderbot blenderbottokenizer default_chat_template a very simple chat template that just adds whitespace between messages
import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot import BlenderbotTokenizer


logger = logging.get_logger(__name__)


VOCAB_FILES_NAMES = {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
    "merges_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
    "tokenizer_config_file": {
        "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"facebook/blenderbot-3B": 128}


class BlenderbotTokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = BlenderbotTokenizer

    
    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        errors="replace",
        bos_token="<s>",
        eos_token="</s>",
        sep_token="</s>",
        cls_token="<s>",
        unk_token="<unk>",
        pad_token="<pad>",
        mask_token="<mask>",
        add_prefix_space=False,
        trim_offsets=True,
        **kwargs,
    ):
        mask_token = (
            AddedToken(mask_token, lstrip=True, rstrip=False, normalized=False)
            if isinstance(mask_token, str)
            else mask_token
        )
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            errors=errors,
            bos_token=bos_token,
            eos_token=eos_token,
            sep_token=sep_token,
            cls_token=cls_token,
            unk_token=unk_token,
            pad_token=pad_token,
            mask_token=mask_token,
            add_prefix_space=add_prefix_space,
            trim_offsets=trim_offsets,
            **kwargs,
        )

        pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
        if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
            pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
            pre_tok_state["add_prefix_space"] = add_prefix_space
            self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

        self.add_prefix_space = add_prefix_space

        tokenizer_component = "post_processor"
        tokenizer_component_instance = getattr(self.backend_tokenizer, tokenizer_component, None)
        if tokenizer_component_instance:
            state = json.loads(tokenizer_component_instance.__getstate__())

            
            if "sep" in state:
                state["sep"] = tuple(state["sep"])
            if "cls" in state:
                state["cls"] = tuple(state["cls"])

            changes_to_apply = False

            if state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
                state["add_prefix_space"] = add_prefix_space
                changes_to_apply = True

            if state.get("trim_offsets", trim_offsets) != trim_offsets:
                state["trim_offsets"] = trim_offsets
                changes_to_apply = True

            if changes_to_apply:
                component_class = getattr(processors, state.pop("type"))
                new_value = component_class(**state)
                setattr(self.backend_tokenizer, tokenizer_component, new_value)

    @property
    
    def mask_token(self) -> str:
        
        if self._mask_token is None:
            if self.verbose:
                logger.error("Using mask_token, but it is not set yet.")
            return None
        return str(self._mask_token)

    @mask_token.setter
    def mask_token(self, value):
        
        
        
        value = AddedToken(value, lstrip=True, rstrip=False) if isinstance(value, str) else value
        self._mask_token = value

    
    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)
        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*args, **kwargs)

    
    def _encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)

        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*args, **kwargs)

    
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)

    
    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

    def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None):
        
        return token_ids_0 + [self.eos_token_id]

    @property
    
    def default_chat_template(self):
        
        logger.warning_once(
            "\nNo chat template is defined for this tokenizer - using the default template "
            f"for the {self.__class__.__name__} class. If the default is not appropriate for "
            "your model, please set `tokenizer.chat_template` to an appropriate template. "
            "See https://huggingface.co/docs/transformers/main/chat_templating for more information.\n"
        )
        return (
            "{% for message in messages %}"
            "{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}"
            "{{ message['content'] }}"
            "{% if not loop.last %}{{ '  ' }}{% endif %}"
            "{% endfor %}"
            "{{ eos_token }}"
        )