codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for bloom import pickle from typing import optional tuple from tokenizationutilsbase import batchencoding from tokenizationutilsfast import pretrainedtokenizerfast from utils import logging logger logging getloggername vocabfilesnames tokenizerfile tokenizer json pretrainedvocabfilesmap tokenizerfile bigsciencetokenizer https huggingface cobigsciencetokenizerblobmaintokenizer json bigsciencebloom560m https huggingface cobigsciencebloom560mblobmaintokenizer json bigsciencebloom1b1 https huggingface cobigsciencebloom1b1blobmaintokenizer json bigsciencebloom1b7 https huggingface cobigsciencebloom1b7blobmaintokenizer json bigsciencebloom3b https huggingface cobigsciencebloom3bblobmaintokenizer json bigsciencebloom7b1 https huggingface cobigsciencebloom7b1blobmaintokenizer json bigsciencebloom https huggingface cobigsciencebloomblobmaintokenizer json class bloomtokenizerfastpretrainedtokenizerfast vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap modelinputnames inputids attentionmask slowtokenizerclass none no maxmodelinputsizes as bloom uses alibi positional embeddings def init self vocabfilenone mergesfilenone tokenizerfilenone unktokenunk bostokens eostokens padtokenpad addprefixspacefalse cleanuptokenizationspacesfalse kwargs super init vocabfile mergesfile tokenizerfiletokenizerfile unktokenunktoken bostokenbostoken eostokeneostoken padtokenpadtoken addprefixspaceaddprefixspace cleanuptokenizationspacescleanuptokenizationspaces kwargs todo arthurzucker this can only work one way for now to update lateron tests should also properly check this as they were green before pretokstate pickle dumpsself backendtokenizer pretokenizer decoderstate pickle dumpsself backendtokenizer decoder if addprefixspace pretokstate pretokstate replaceb addprefixspace false b addprefixspace true decoderstate decoderstate replaceb addprefixspace false b addprefixspace true self backendtokenizer pretokenizer pickle loadspretokstate self backendtokenizer decoder pickle loadsdecoderstate self addprefixspace addprefixspace def batchencodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false if not self addprefixspace or not issplitintowords raise exception fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super batchencodeplusargs kwargs def encodeplusself args kwargs batchencoding issplitintowords kwargs getissplitintowords false if not self addprefixspace or not issplitintowords raise exception fyou need to instantiate self class name with addprefixspacetrue to use it with pretokenized inputs return super encodeplusargs kwargs def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles property copied from transformers models gpt2 tokenizationgpt2 gpt2tokenizer defaultchattemplate def defaultchattemplateself logger warningonce nno chat template is defined for this tokenizer using the default template ffor the self class name class if the default is not appropriate for your model please set tokenizer chattemplate to an appropriate template see https huggingface codocstransformersmainchattemplating for more information n return for message in messages message content eostoken endfor coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for bloom construct a fast bloom tokenizer backed by huggingface s tokenizers library based on byte level byte pair encoding this tokenizer has been trained to treat spaces like parts of the tokens a bit like sentencepiece so a word will be encoded differently whether it is at the beginning of the sentence without space or not python from transformers import bloomtokenizerfast tokenizer bloomtokenizerfast from_pretrained bigscience bloom tokenizer hello world input_ids 59414 8876 tokenizer hello world input_ids 86153 8876 you can get around that behavior by passing add_prefix_space true when instantiating this tokenizer but since the model was not pretrained this way it might yield a decrease in performance tip when used with is_split_into_words true this tokenizer needs to be instantiated with add_prefix_space true tip this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str path to the vocabulary file merges_file str path to the merges file errors str optional defaults to replace paradigm to follow when decoding bytes to utf 8 see bytes decode https docs python org 3 library stdtypes html bytes decode for more information unk_token str optional defaults to endoftext the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead bos_token str optional defaults to endoftext the beginning of sequence token eos_token str optional defaults to endoftext the end of sequence token add_prefix_space bool optional defaults to false whether or not to add an initial space to the input this allows to treat the leading word just as any other word bloom tokenizer detect beginning of words by the preceding space trim_offsets bool optional defaults to true whether or not the post processing step should trim offsets to avoid including whitespaces no max_model_input_sizes as bloom uses alibi positional embeddings todo arthurzucker this can only work one way for now to update later on tests should also properly check this as they were green before copied from transformers models gpt2 tokenization_gpt2 gpt2tokenizer default_chat_template a simple chat template that ignores role information and just concatenates messages with eos tokens
import pickle
from typing import Optional, Tuple

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "tokenizer_file": {
        "bigscience/tokenizer": "https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json",
        "bigscience/bloom-560m": "https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json",
        "bigscience/bloom-1b1": "https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json",
        "bigscience/bloom-1b7": "https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json",
        "bigscience/bloom-3b": "https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json",
        "bigscience/bloom-7b1": "https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json",
        "bigscience/bloom": "https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json",
    },
}


class BloomTokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = None
    

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        unk_token="<unk>",
        bos_token="<s>",
        eos_token="</s>",
        pad_token="<pad>",
        add_prefix_space=False,
        clean_up_tokenization_spaces=False,
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            pad_token=pad_token,
            add_prefix_space=add_prefix_space,
            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
            **kwargs,
        )
        
        
        pre_tok_state = pickle.dumps(self.backend_tokenizer.pre_tokenizer)
        decoder_state = pickle.dumps(self.backend_tokenizer.decoder)

        if add_prefix_space:
            pre_tok_state = pre_tok_state.replace(b'"add_prefix_space":false', b'"add_prefix_space": true')
            decoder_state = decoder_state.replace(b'"add_prefix_space":false', b'"add_prefix_space": true')
        self.backend_tokenizer.pre_tokenizer = pickle.loads(pre_tok_state)
        self.backend_tokenizer.decoder = pickle.loads(decoder_state)

        self.add_prefix_space = add_prefix_space

    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)
        if not (self.add_prefix_space or not is_split_into_words):
            raise Exception(
                f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
                " pretokenized inputs."
            )

        return super()._batch_encode_plus(*args, **kwargs)

    def _encode_plus(self, *args, **kwargs) -> BatchEncoding:
        is_split_into_words = kwargs.get("is_split_into_words", False)

        if not (self.add_prefix_space or not is_split_into_words):
            raise Exception(
                f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
                " pretokenized inputs."
            )

        return super()._encode_plus(*args, **kwargs)

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)

    @property
    
    def default_chat_template(self):
        
        logger.warning_once(
            "\nNo chat template is defined for this tokenizer - using the default template "
            f"for the {self.__class__.__name__} class. If the default is not appropriate for "
            "your model, please set `tokenizer.chat_template` to an appropriate template. "
            "See https://huggingface.co/docs/transformers/main/chat_templating for more information.\n"
        )
        return "{% for message in messages %}" "{{ message.content }}{{ eos_token }}" "{% endfor %}"