codingutf8 2022 eleutherai and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for gptneox import json from typing import optional tuple from tokenizers import pretokenizers from tokenizationutilsfast import pretrainedtokenizerfast from utils import logging logger logging getloggername vocabfilesnames vocabfile vocab json mergesfile merges txt tokenizerfile tokenizer json pretrainedvocabfilesmap tokenizerfile eleutheraigptneox20b https huggingface coeleutheraigptneox20bresolvemaintokenizer json pretrainedpositionalembeddingssizes gptneox20b 2048 class gptneoxtokenizerfastpretrainedtokenizerfast vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask def init self vocabfilenone mergesfilenone tokenizerfilenone unktokenendoftext bostokenendoftext eostokenendoftext addprefixspacefalse kwargs super init vocabfile mergesfile tokenizerfiletokenizerfile unktokenunktoken bostokenbostoken eostokeneostoken addprefixspaceaddprefixspace kwargs pretokstate json loadsself backendtokenizer pretokenizer getstate if pretokstate getaddprefixspace addprefixspace addprefixspace pretokclass getattrpretokenizers pretokstate poptype pretokstateaddprefixspace addprefixspace self backendtokenizer pretokenizer pretokclasspretokstate self addprefixspace addprefixspace def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles property copied from transformers models gpt2 tokenizationgpt2 gpt2tokenizer defaultchattemplate def defaultchattemplateself logger warningonce nno chat template is defined for this tokenizer using the default template ffor the self class name class if the default is not appropriate for your model please set tokenizer chattemplate to an appropriate template see https huggingface codocstransformersmainchattemplating for more information n return for message in messages message content eostoken endfor coding utf 8 2022 eleutherai and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for gptneox construct a fast gpt neox 20b tokenizer backed by huggingface s tokenizers library based on byte level byte pair encoding this tokenizer has been trained to treat spaces like parts of the tokens a bit like sentencepiece so a word will be encoded differently whether it is at the beginning of the sentence without space or not python from transformers import gptneoxtokenizerfast tokenizer gptneoxtokenizerfast from_pretrained gpt2 tokenizer hello world input_ids 15496 995 tokenizer hello world input_ids 18435 995 you can get around that behavior by passing add_prefix_space true when instantiating this tokenizer but since the model was not pretrained this way it might yield a decrease in performance tip when used with is_split_into_words true this tokenizer needs to be instantiated with add_prefix_space true tip this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str path to the vocabulary file merges_file str path to the merges file errors str optional defaults to replace paradigm to follow when decoding bytes to utf 8 see bytes decode https docs python org 3 library stdtypes html bytes decode for more information unk_token str optional defaults to endoftext the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead bos_token str optional defaults to endoftext the beginning of sequence token eos_token str optional defaults to endoftext the end of sequence token add_prefix_space bool optional defaults to false whether or not to add an initial space to the input this allows to treat the leading word just as any other word gptneox tokenizer detect beginning of words by the preceding space trim_offsets bool optional defaults to true whether or not the post processing step should trim offsets to avoid including whitespaces copied from transformers models gpt2 tokenization_gpt2 gpt2tokenizer default_chat_template a simple chat template that ignores role information and just concatenates messages with eos tokens
import json
from typing import Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "tokenizer_file": {
        "EleutherAI/gpt-neox-20b": "https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json",
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "gpt-neox-20b": 2048,
}


class GPTNeoXTokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        unk_token="<|endoftext|>",
        bos_token="<|endoftext|>",
        eos_token="<|endoftext|>",
        add_prefix_space=False,
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            add_prefix_space=add_prefix_space,
            **kwargs,
        )

        pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
        if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
            pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
            pre_tok_state["add_prefix_space"] = add_prefix_space
            self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

        self.add_prefix_space = add_prefix_space

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)

    @property
    
    def default_chat_template(self):
        
        logger.warning_once(
            "\nNo chat template is defined for this tokenizer - using the default template "
            f"for the {self.__class__.__name__} class. If the default is not appropriate for "
            "your model, please set `tokenizer.chat_template` to an appropriate template. "
            "See https://huggingface.co/docs/transformers/main/chat_templating for more information.\n"
        )
        return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"