codingutf8 2021 the open ai team s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for openai gpt from typing import list optional tuple from tokenizers import pretokenizers from tokenizationutilsfast import pretrainedtokenizerfast from utils import logging from tokenizationclip import cliptokenizer logger logging getloggername vocabfilesnames vocabfile vocab json mergesfile merges txt tokenizerfile tokenizer json pretrainedvocabfilesmap vocabfile openaiclipvitbasepatch32 https huggingface coopenaiclipvitbasepatch32resolvemainvocab json mergesfile openaiclipvitbasepatch32 https huggingface coopenaiclipvitbasepatch32resolvemainmerges txt tokenizerfile openaiclipvitbasepatch32 https huggingface coopenaiclipvitbasepatch32resolvemaintokenizer json pretrainedpositionalembeddingssizes openaiclipvitbasepatch32 77 class cliptokenizerfastpretrainedtokenizerfast vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask slowtokenizerclass cliptokenizer def init self vocabfilenone mergesfilenone tokenizerfilenone unktokenendoftext bostokenstartoftext eostokenendoftext padtokenendoftext hack to enable padding kwargs super init vocabfile mergesfile tokenizerfiletokenizerfile unktokenunktoken bostokenbostoken eostokeneostoken padtokenpadtoken kwargs if not isinstanceself backendtokenizer pretokenizer pretokenizers sequence raise valueerror the backendtokenizer provided does not match the expected format the clip tokenizer has been heavily modified from transformers version 4 17 0 you need to convert the tokenizer you are using to be compatible with this version the easiest way to do so is cliptokenizerfast frompretrainedpathtolocalfolderorhubrepo fromslowtrue if you want to use your existing tokenizer you will have to revert to a version prior to 4 17 0 of transformers self wrapdecodemethodbackendtokenizer very ugly hack to enable padding to have a correct decoding see https github comhuggingfacetokenizersissues872 def wrapdecodemethodbackendtokenizerself origdecodemethod self backendtokenizer decode def newdecodemethodargs kwargs text origdecodemethodargs kwargs text text replaceself backendtokenizer model endofwordsuffix strip return text self backendtokenizer decode newdecodemethod def buildinputswithspecialtokens self tokenids0 listint tokenids1 optionallistint none listint bostoken self bostokenid eostoken self eostokenid if tokenids1 is none return bostoken tokenids0 eostoken return bostoken tokenids0 eostoken eostoken tokenids1 eostoken def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint bostoken self bostokenid eostoken self eostokenid if tokenids1 is none return lenbostoken tokenids0 eostoken 0 return lenbostoken tokenids0 eostoken eostoken tokenids1 eostoken 0 def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles coding utf 8 2021 the open ai team s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for openai gpt construct a fast clip tokenizer backed by huggingface s tokenizers library based on byte level byte pair encoding this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str optional path to the vocabulary file merges_file str optional path to the merges file tokenizer_file str optional the path to a tokenizer file to use instead of the vocab file unk_token str optional defaults to endoftext the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead bos_token str optional defaults to startoftext the beginning of sequence token eos_token str optional defaults to endoftext the end of sequence token pad_token str optional defaults to endoftext the token used for padding for example when batching sequences of different lengths hack to enable padding very ugly hack to enable padding to have a correct decoding see https github com huggingface tokenizers issues 872 build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a clip sequence has the following format single sequence startoftext x endoftext pairs of sequences are not the expected use case but they will be handled without a separator args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens create a mask from the two sequences passed clip does not make use of token type ids therefore a list of zeros is returned args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of zeros
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "openai/clip-vit-base-patch32": "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json",
    },
    "merges_file": {
        "openai/clip-vit-base-patch32": "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "openai/clip-vit-base-patch32": (
            "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"
        ),
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "openai/clip-vit-base-patch32": 77,
}


class CLIPTokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = CLIPTokenizer

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        unk_token="<|endoftext|>",
        bos_token="<|startoftext|>",
        eos_token="<|endoftext|>",
        pad_token="<|endoftext|>",  
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            pad_token=pad_token,
            **kwargs,
        )

        if not isinstance(self.backend_tokenizer.pre_tokenizer, pre_tokenizers.Sequence):
            raise ValueError(
                "The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been"
                " heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using"
                " to be compatible with this version.The easiest way to do so is"
                ' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'
                " to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of"
                " transformers."
            )

        self._wrap_decode_method_backend_tokenizer()

    
    def _wrap_decode_method_backend_tokenizer(self):
        orig_decode_method = self.backend_tokenizer.decode

        def new_decode_method(*args, **kwargs):
            text = orig_decode_method(*args, **kwargs)
            text = text.replace(self.backend_tokenizer.model.end_of_word_suffix, " ").strip()
            return text

        self.backend_tokenizer.decode = new_decode_method

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        bos_token = [self.bos_token_id]
        eos_token = [self.eos_token_id]

        if token_ids_1 is None:
            return bos_token + token_ids_0 + eos_token
        return bos_token + token_ids_0 + eos_token + eos_token + token_ids_1 + eos_token

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        bos_token = [self.bos_token_id]
        eos_token = [self.eos_token_id]

        if token_ids_1 is None:
            return len(bos_token + token_ids_0 + eos_token) * [0]
        return len(bos_token + token_ids_0 + eos_token + eos_token + token_ids_1 + eos_token) * [0]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)