codingutf8 2018 the open ai team s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for openai gpt import json from typing import optional tuple from tokenizers import pretokenizers from tokenizationutilsbase import batchencoding from tokenizationutilsfast import pretrainedtokenizerfast from utils import logging from tokenizationgpt2 import gpt2tokenizer logger logging getloggername vocabfilesnames vocabfile vocab json mergesfile merges txt tokenizerfile tokenizer json pretrainedvocabfilesmap vocabfile gpt2 https huggingface cogpt2resolvemainvocab json gpt2medium https huggingface cogpt2mediumresolvemainvocab json gpt2large https huggingface cogpt2largeresolvemainvocab json gpt2xl https huggingface cogpt2xlresolvemainvocab json distilgpt2 https huggingface codistilgpt2resolvemainvocab json mergesfile gpt2 https huggingface cogpt2resolvemainmerges txt gpt2medium https huggingface cogpt2mediumresolvemainmerges txt gpt2large https huggingface cogpt2largeresolvemainmerges txt gpt2xl https huggingface cogpt2xlresolvemainmerges txt distilgpt2 https huggingface codistilgpt2resolvemainmerges txt tokenizerfile gpt2 https huggingface cogpt2resolvemaintokenizer json gpt2medium https huggingface cogpt2mediumresolvemaintokenizer json gpt2large https huggingface cogpt2largeresolvemaintokenizer json gpt2xl https huggingface cogpt2xlresolvemaintokenizer json distilgpt2 https huggingface codistilgpt2resolvemaintokenizer json pretrainedpositionalembeddingssizes gpt2 1024 gpt2medium 1024 gpt2large 1024 gpt2xl 1024 distilgpt2 1024 class gpt2tokenizerfastpretrainedtokenizerfast vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask slowtokenizerclass gpt2tokenizer def init self vocabfilenone mergesfilenone tokenizerfilenone unktokenendoftext bostokenendoftext eostokenendoftext addprefixspacefalse kwargs super init vocabfile mergesfile tokenizerfiletokenizerfile unktokenunktoken bostokenbostoken eostokeneostoken addprefixspaceaddprefixspace kwargs self addbostoken kwargs popaddbostoken false pretokstate json loadsself backendtokenizer pretokenizer getstate if pretokstate getaddprefixspace addprefixspace addprefixspace pretokclass getattrpretokenizers pretokstate poptype pretokstateaddprefixspace addprefixspace self backendtokenizer pretokenizer pretokclasspretokstate self addprefixspace addprefixspace def batchencodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false assert self addprefixspace or not issplitintowords fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super batchencodeplusargs kwargs def encodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false assert self addprefixspace or not issplitintowords fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super encodeplusargs kwargs def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles property copied from transformers models gpt2 tokenizationgpt2 gpt2tokenizer defaultchattemplate def defaultchattemplateself logger warningonce nno chat template is defined for this tokenizer using the default template ffor the self class name class if the default is not appropriate for your model please set tokenizer chattemplate to an appropriate template see https huggingface codocstransformersmainchattemplating for more information n return for message in messages message content eostoken endfor coding utf 8 2018 the open ai team s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for openai gpt construct a fast gpt 2 tokenizer backed by huggingface s tokenizers library based on byte level byte pair encoding this tokenizer has been trained to treat spaces like parts of the tokens a bit like sentencepiece so a word will be encoded differently whether it is at the beginning of the sentence without space or not python from transformers import gpt2tokenizerfast tokenizer gpt2tokenizerfast from_pretrained gpt2 tokenizer hello world input_ids 15496 995 tokenizer hello world input_ids 18435 995 you can get around that behavior by passing add_prefix_space true when instantiating this tokenizer but since the model was not pretrained this way it might yield a decrease in performance tip when used with is_split_into_words true this tokenizer needs to be instantiated with add_prefix_space true tip this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str optional path to the vocabulary file merges_file str optional path to the merges file tokenizer_file str optional path to tokenizers https github com huggingface tokenizers file generally has a json extension that contains everything needed to load the tokenizer unk_token str optional defaults to endoftext the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead bos_token str optional defaults to endoftext the beginning of sequence token eos_token str optional defaults to endoftext the end of sequence token add_prefix_space bool optional defaults to false whether or not to add an initial space to the input this allows to treat the leading word just as any other word gpt2 tokenizer detect beginning of words by the preceding space copied from transformers models gpt2 tokenization_gpt2 gpt2tokenizer default_chat_template a simple chat template that ignores role information and just concatenates messages with eos tokens
import json
from typing import Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpt2 import GPT2Tokenizer


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "gpt2": 1024,
    "gpt2-medium": 1024,
    "gpt2-large": 1024,
    "gpt2-xl": 1024,
    "distilgpt2": 1024,
}


class GPT2TokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = GPT2Tokenizer

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        unk_token="<|endoftext|>",
        bos_token="<|endoftext|>",
        eos_token="<|endoftext|>",
        add_prefix_space=False,
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            add_prefix_space=add_prefix_space,
            **kwargs,
        )

        self.add_bos_token = kwargs.pop("add_bos_token", False)

        pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
        if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
            pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
            pre_tok_state["add_prefix_space"] = add_prefix_space
            self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

        self.add_prefix_space = add_prefix_space

    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)
        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*args, **kwargs)

    def _encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)

        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*args, **kwargs)

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)

    @property
    
    def default_chat_template(self):
        
        logger.warning_once(
            "\nNo chat template is defined for this tokenizer - using the default template "
            f"for the {self.__class__.__name__} class. If the default is not appropriate for "
            "your model, please set `tokenizer.chat_template` to an appropriate template. "
            "See https://huggingface.co/docs/transformers/main/chat_templating for more information.\n"
        )
        return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"