codingutf8 2018 the open ai team s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license fast tokenization classes for roberta import json from typing import list optional tuple from tokenizers import pretokenizers processors from tokenizationutilsbase import addedtoken batchencoding from tokenizationutilsfast import pretrainedtokenizerfast from utils import logging from tokenizationroberta import robertatokenizer logger logging getloggername vocabfilesnames vocabfile vocab json mergesfile merges txt tokenizerfile tokenizer json pretrainedvocabfilesmap vocabfile robertabase https huggingface corobertabaseresolvemainvocab json robertalarge https huggingface corobertalargeresolvemainvocab json robertalargemnli https huggingface corobertalargemnliresolvemainvocab json distilrobertabase https huggingface codistilrobertabaseresolvemainvocab json robertabaseopenaidetector https huggingface corobertabaseopenaidetectorresolvemainvocab json robertalargeopenaidetector https huggingface corobertalargeopenaidetectorresolvemainvocab json mergesfile robertabase https huggingface corobertabaseresolvemainmerges txt robertalarge https huggingface corobertalargeresolvemainmerges txt robertalargemnli https huggingface corobertalargemnliresolvemainmerges txt distilrobertabase https huggingface codistilrobertabaseresolvemainmerges txt robertabaseopenaidetector https huggingface corobertabaseopenaidetectorresolvemainmerges txt robertalargeopenaidetector https huggingface corobertalargeopenaidetectorresolvemainmerges txt tokenizerfile robertabase https huggingface corobertabaseresolvemaintokenizer json robertalarge https huggingface corobertalargeresolvemaintokenizer json robertalargemnli https huggingface corobertalargemnliresolvemaintokenizer json distilrobertabase https huggingface codistilrobertabaseresolvemaintokenizer json robertabaseopenaidetector https huggingface corobertabaseopenaidetectorresolvemaintokenizer json robertalargeopenaidetector https huggingface corobertalargeopenaidetectorresolvemaintokenizer json pretrainedpositionalembeddingssizes robertabase 512 robertalarge 512 robertalargemnli 512 distilrobertabase 512 robertabaseopenaidetector 512 robertalargeopenaidetector 512 class robertatokenizerfastpretrainedtokenizerfast vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask slowtokenizerclass robertatokenizer def init self vocabfilenone mergesfilenone tokenizerfilenone errorsreplace bostokens eostokens septokens clstokens unktokenunk padtokenpad masktokenmask addprefixspacefalse trimoffsetstrue kwargs masktoken addedtokenmasktoken lstriptrue rstripfalse normalizedfalse if isinstancemasktoken str else masktoken super init vocabfile mergesfile tokenizerfiletokenizerfile errorserrors bostokenbostoken eostokeneostoken septokenseptoken clstokenclstoken unktokenunktoken padtokenpadtoken masktokenmasktoken addprefixspaceaddprefixspace trimoffsetstrimoffsets kwargs pretokstate json loadsself backendtokenizer pretokenizer getstate if pretokstate getaddprefixspace addprefixspace addprefixspace pretokclass getattrpretokenizers pretokstate poptype pretokstateaddprefixspace addprefixspace self backendtokenizer pretokenizer pretokclasspretokstate self addprefixspace addprefixspace tokenizercomponent postprocessor tokenizercomponentinstance getattrself backendtokenizer tokenizercomponent none if tokenizercomponentinstance state json loadstokenizercomponentinstance getstate the lists sep and cls must be cased in tuples for the object postprocessorclass if sep in state statesep tuplestatesep if cls in state statecls tuplestatecls changestoapply false if state getaddprefixspace addprefixspace addprefixspace stateaddprefixspace addprefixspace changestoapply true if state gettrimoffsets trimoffsets trimoffsets statetrimoffsets trimoffsets changestoapply true if changestoapply componentclass getattrprocessors state poptype newvalue componentclassstate setattrself backendtokenizer tokenizercomponent newvalue property def masktokenself str if self masktoken is none if self verbose logger errorusing masktoken but it is not set yet return none return strself masktoken masktoken setter def masktokenself value mask token behave like a normal word i e include the space before it so we set lstrip to true value addedtokenvalue lstriptrue rstripfalse if isinstancevalue str else value self masktoken value def batchencodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false assert self addprefixspace or not issplitintowords fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super batchencodeplusargs kwargs def encodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false assert self addprefixspace or not issplitintowords fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super encodeplusargs kwargs def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles def buildinputswithspecialtokensself tokenids0 tokenids1none output self bostokenid tokenids0 self eostokenid if tokenids1 is none return output return output self eostokenid tokenids1 self eostokenid def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint sep self septokenid cls self clstokenid if tokenids1 is none return lencls tokenids0 sep 0 return lencls tokenids0 sep sep tokenids1 sep 0 coding utf 8 2018 the open ai team s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license fast tokenization classes for roberta construct a fast roberta tokenizer backed by huggingface s tokenizers library derived from the gpt 2 tokenizer using byte level byte pair encoding this tokenizer has been trained to treat spaces like parts of the tokens a bit like sentencepiece so a word will be encoded differently whether it is at the beginning of the sentence without space or not python from transformers import robertatokenizerfast tokenizer robertatokenizerfast from_pretrained roberta base tokenizer hello world input_ids 0 31414 232 2 tokenizer hello world input_ids 0 20920 232 2 you can get around that behavior by passing add_prefix_space true when instantiating this tokenizer or when you call it on some text but since the model was not pretrained this way it might yield a decrease in performance tip when used with is_split_into_words true this tokenizer needs to be instantiated with add_prefix_space true tip this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str path to the vocabulary file merges_file str path to the merges file errors str optional defaults to replace paradigm to follow when decoding bytes to utf 8 see bytes decode https docs python org 3 library stdtypes html bytes decode for more information bos_token str optional defaults to s the beginning of sequence token that was used during pretraining can be used a sequence classifier token tip when building a sequence using special tokens this is not the token that is used for the beginning of sequence the token used is the cls_token tip eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip sep_token str optional defaults to s the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens cls_token str optional defaults to s the classifier token which is used when doing sequence classification classification of the whole sequence instead of per token classification it is the first token of the sequence when built with special tokens unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths mask_token str optional defaults to mask the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict add_prefix_space bool optional defaults to false whether or not to add an initial space to the input this allows to treat the leading word just as any other word roberta tokenizer detect beginning of words by the preceding space trim_offsets bool optional defaults to true whether the post processing step should trim offsets to avoid including whitespaces the lists sep and cls must be cased in tuples for the object post_processor_class str mask token to use when training a model with masked language modeling log an error if used while not having been set roberta tokenizer has a special mask token to be usable in the fill mask pipeline the mask token will greedily comprise the space before the mask overriding the default behavior of the mask token to have it eat the space before it this is needed to preserve backward compatibility with all the previously used models based on roberta mask token behave like a normal word i e include the space before it so we set lstrip to true create a mask from the two sequences passed to be used in a sequence pair classification task roberta does not make use of token type ids therefore a list of zeros is returned args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of zeros
import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/vocab.json",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/merges.txt",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"
        ),
    },
    "tokenizer_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/tokenizer.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/tokenizer.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json",
        "roberta-base-openai-detector": (
            "https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"
        ),
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"
        ),
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "roberta-base": 512,
    "roberta-large": 512,
    "roberta-large-mnli": 512,
    "distilroberta-base": 512,
    "roberta-base-openai-detector": 512,
    "roberta-large-openai-detector": 512,
}


class RobertaTokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = RobertaTokenizer

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        errors="replace",
        bos_token="<s>",
        eos_token="</s>",
        sep_token="</s>",
        cls_token="<s>",
        unk_token="<unk>",
        pad_token="<pad>",
        mask_token="<mask>",
        add_prefix_space=False,
        trim_offsets=True,
        **kwargs,
    ):
        mask_token = (
            AddedToken(mask_token, lstrip=True, rstrip=False, normalized=False)
            if isinstance(mask_token, str)
            else mask_token
        )
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            errors=errors,
            bos_token=bos_token,
            eos_token=eos_token,
            sep_token=sep_token,
            cls_token=cls_token,
            unk_token=unk_token,
            pad_token=pad_token,
            mask_token=mask_token,
            add_prefix_space=add_prefix_space,
            trim_offsets=trim_offsets,
            **kwargs,
        )

        pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
        if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
            pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
            pre_tok_state["add_prefix_space"] = add_prefix_space
            self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

        self.add_prefix_space = add_prefix_space

        tokenizer_component = "post_processor"
        tokenizer_component_instance = getattr(self.backend_tokenizer, tokenizer_component, None)
        if tokenizer_component_instance:
            state = json.loads(tokenizer_component_instance.__getstate__())

            
            if "sep" in state:
                state["sep"] = tuple(state["sep"])
            if "cls" in state:
                state["cls"] = tuple(state["cls"])

            changes_to_apply = False

            if state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
                state["add_prefix_space"] = add_prefix_space
                changes_to_apply = True

            if state.get("trim_offsets", trim_offsets) != trim_offsets:
                state["trim_offsets"] = trim_offsets
                changes_to_apply = True

            if changes_to_apply:
                component_class = getattr(processors, state.pop("type"))
                new_value = component_class(**state)
                setattr(self.backend_tokenizer, tokenizer_component, new_value)

    @property
    def mask_token(self) -> str:
        
        if self._mask_token is None:
            if self.verbose:
                logger.error("Using mask_token, but it is not set yet.")
            return None
        return str(self._mask_token)

    @mask_token.setter
    def mask_token(self, value):
        
        
        
        value = AddedToken(value, lstrip=True, rstrip=False) if isinstance(value, str) else value
        self._mask_token = value

    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)
        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*args, **kwargs)

    def _encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)

        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*args, **kwargs)

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        output = [self.bos_token_id] + token_ids_0 + [self.eos_token_id]
        if token_ids_1 is None:
            return output

        return output + [self.eos_token_id] + token_ids_1 + [self.eos_token_id]

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]