this is an ingraph tokenizer for gpt2 it should be initialized similarly to other tokenizers using the frompretrained method it can also be initialized with the fromtokenizer method which imports settings from an existing standard tokenizer object ingraph tokenizers unlike other hugging face tokenizers are actually keras layers and are designed to be run when the model is called rather than during preprocessing as a result they have somewhat more limited options than standard tokenizer classes they are most useful when you want to create an endtoend model that goes straight from tf string inputs to outputs args vocab dictstr int vocabulary dict for byte pair tokenizer merges liststr merges list for byte pair tokenizer creates tfgpt2tokenizer from gpt2tokenizer args tokenizer gpt2tokenizer examples python from transformers import autotokenizer tfgpt2tokenizer tokenizer autotokenizer frompretrainedgpt2 tftokenizer tfgpt2tokenizer fromtokenizertokenizer creates tfgpt2tokenizer from pretrained gpt2tokenizer args pretrainedmodelnameorpath unionstr os pathlike path to pretrained model examples python from transformers import tfgpt2tokenizer tftokenizer tfgpt2tokenizer frompretrainedgpt2 creates tfgpt2tokenizer from configurations args config dict dictionary with keys such as stated in getconfig pad the tokens up to max length this is an in graph tokenizer for gpt2 it should be initialized similarly to other tokenizers using the from_pretrained method it can also be initialized with the from_tokenizer method which imports settings from an existing standard tokenizer object in graph tokenizers unlike other hugging face tokenizers are actually keras layers and are designed to be run when the model is called rather than during preprocessing as a result they have somewhat more limited options than standard tokenizer classes they are most useful when you want to create an end to end model that goes straight from tf string inputs to outputs args vocab dict str int vocabulary dict for byte pair tokenizer merges list str merges list for byte pair tokenizer creates tfgpt2tokenizer from gpt2tokenizer args tokenizer gpt2tokenizer examples python from transformers import autotokenizer tfgpt2tokenizer tokenizer autotokenizer from_pretrained gpt2 tf_tokenizer tfgpt2tokenizer from_tokenizer tokenizer creates tfgpt2tokenizer from pretrained gpt2tokenizer args pretrained_model_name_or_path union str os pathlike path to pretrained model examples python from transformers import tfgpt2tokenizer tf_tokenizer tfgpt2tokenizer from_pretrained gpt2 creates tfgpt2tokenizer from configurations args config dict dictionary with keys such as stated in get_config pad the tokens up to max length
import os
from typing import Dict, List, Union

import tensorflow as tf
from keras_nlp.tokenizers import BytePairTokenizer
from tensorflow_text import pad_model_inputs

from .tokenization_gpt2 import GPT2Tokenizer


class TFGPT2Tokenizer(tf.keras.layers.Layer):
    

    def __init__(self, vocab: Dict[str, int], merges: List[str], max_length: int = None, pad_token_id: int = None):
        super().__init__()
        self.pad_token_id = pad_token_id
        self.max_length = max_length
        self.vocab = vocab
        self.merges = merges
        self.tf_tokenizer = BytePairTokenizer(vocab, merges, sequence_length=max_length)

    @classmethod
    def from_tokenizer(cls, tokenizer: GPT2Tokenizer, *args, **kwargs):
        
        merges = [" ".join(m) for m in tokenizer.bpe_ranks.keys()]
        vocab = tokenizer.get_vocab()
        return cls(vocab, merges, *args, **kwargs)

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], *init_inputs, **kwargs):
        
        tokenizer = GPT2Tokenizer.from_pretrained(pretrained_model_name_or_path, *init_inputs, **kwargs)
        return cls.from_tokenizer(tokenizer, *init_inputs, **kwargs)

    @classmethod
    def from_config(cls, config):
        
        return cls(**config)

    def get_config(self):
        return {
            "vocab": self.vocab,
            "merges": self.merges,
            "max_length": self.max_length,
            "pad_token_id": self.pad_token_id,
        }

    def call(self, x, max_length: int = None):
        input_ids = self.tf_tokenizer(x)
        attention_mask = tf.ones_like(input_ids)

        if self.pad_token_id is not None:
            
            max_length = max_length if max_length is not None else self.max_length

            if max_length is not None:
                input_ids, attention_mask = pad_model_inputs(
                    input_ids, max_seq_length=max_length, pad_value=self.pad_token_id
                )

        return {"attention_mask": attention_mask, "input_ids": input_ids}