'''simple docstring'''






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


__a										=					logging.get_logger(__name__)

__a										=					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__a										=					{
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

__a										=					{
    """gpt2""": 1_024,
    """gpt2-medium""": 1_024,
    """gpt2-large""": 1_024,
    """gpt2-xl""": 1_024,
    """distilgpt2""": 1_024,
}





class     A__							(       UpperCAmelCase__   ):





   """simple docstring"""





   UpperCamelCase_       :						Dict    				=     VOCAB_FILES_NAMES
   UpperCamelCase_       :						Any    				=     PRETRAINED_VOCAB_FILES_MAP
   UpperCamelCase_       :						List[Any]    				=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCamelCase_       :						str    				=     ['''input_ids''', '''attention_mask''']
   UpperCamelCase_       :						Tuple    				=     GPTaTokenizer




   def __init__( self     :		Optional[int]						,     lowerCAmelCase__     :		int=None						,     lowerCAmelCase__     :		List[str]=None						,     lowerCAmelCase__     :		Union[str, Any]=None						,     lowerCAmelCase__     :		Tuple="<|endoftext|>"						,     lowerCAmelCase__     :		str="<|endoftext|>"						,     lowerCAmelCase__     :		Dict="<|endoftext|>"						,     lowerCAmelCase__     :		Tuple=False						,     **lowerCAmelCase__     :		Optional[int]						,     )       ->    List[Any]:







      """simple docstring"""




      super().__init__(
          lowercase_						,     lowercase_						,     tokenizer_file=lowercase_						,     unk_token=lowercase_						,     bos_token=lowercase_						,     eos_token=lowercase_						,     add_prefix_space=lowercase_						,     **lowercase_						,     )

      _UpperCAmelCase							:	Union[str, Any]							=   kwargs.pop("add_bos_token"						,     lowercase_			)

      _UpperCAmelCase							:	Dict							=   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
      if pre_tok_state.get("add_prefix_space"						,     lowercase_			) != add_prefix_space:
         _UpperCAmelCase							:	int							=   getattr(lowercase_						,     pre_tok_state.pop("type"			)			)
         _UpperCAmelCase							:	str							=   add_prefix_space
         _UpperCAmelCase							:	Dict							=   pre_tok_class(**lowercase_			)

      _UpperCAmelCase							:	Optional[Any]							=   add_prefix_space




   def 			_lowerCAmelCase		( self     :		str						,     *lowerCAmelCase__     :		List[Any]						,     **lowerCAmelCase__     :		Optional[Any]			)       ->    Optional[int]:







      """simple docstring"""




      _UpperCAmelCase							:	List[Any]							=   kwargs.get("is_split_into_words"						,     lowercase_			)
      assert self.add_prefix_space or not is_split_into_words, (
          F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*lowercase_						,     **lowercase_			)




   def 			_lowerCAmelCase		( self     :		Optional[int]						,     *lowerCAmelCase__     :		List[str]						,     **lowerCAmelCase__     :		List[Any]			)       ->    Dict:







      """simple docstring"""




      _UpperCAmelCase							:	List[str]							=   kwargs.get("is_split_into_words"						,     lowercase_			)

      assert self.add_prefix_space or not is_split_into_words, (
          F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*lowercase_						,     **lowercase_			)




   def 			_lowerCAmelCase		( self     :		int						,     lowerCAmelCase__     :		str						,     lowerCAmelCase__     :		Optional[str] = None			)       ->    List[str]:







      """simple docstring"""




      _UpperCAmelCase							:	List[str]							=   self._tokenizer.model.save(lowercase_						,     name=lowercase_			)
      return tuple(lowercase_			)




   def 			_lowerCAmelCase		( self     :		int						,     lowerCAmelCase__     :		"Conversation"			)       ->    Any:







      """simple docstring"""




      _UpperCAmelCase							:	Dict							=   []
      for is_user, text in conversation.iter_texts():
         input_ids.extend(self.encode(lowercase_						,     add_special_tokens=lowercase_			) + [self.eos_token_id]			)

      if len(lowercase_			) > self.model_max_length:
         _UpperCAmelCase							:	Any							=   input_ids[-self.model_max_length :]
      return input_ids







"""simple docstring"""






import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible




class 							lowerCAmelCase__				(			unittest.TestCase ):






      '''simple docstring'''

      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		[
              '''safety_checker/pytorch_model.bin''',
              '''safety_checker/model.safetensors''',
              '''vae/diffusion_pytorch_model.bin''',
              '''vae/diffusion_pytorch_model.safetensors''',
              '''text_encoder/pytorch_model.bin''',
              '''text_encoder/model.safetensors''',
              '''unet/diffusion_pytorch_model.bin''',
              '''unet/diffusion_pytorch_model.safetensors''',
          ]
          self.assertTrue(is_safetensors_compatible(lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		[
              '''unet/diffusion_pytorch_model.bin''',
              '''unet/diffusion_pytorch_model.safetensors''',
          ]
          self.assertTrue(is_safetensors_compatible(lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		[
              '''safety_checker/pytorch_model.bin''',
              '''safety_checker/model.safetensors''',
              '''vae/diffusion_pytorch_model.bin''',
              '''vae/diffusion_pytorch_model.safetensors''',
              '''text_encoder/pytorch_model.bin''',
              '''text_encoder/model.safetensors''',
              '''unet/diffusion_pytorch_model.bin''',
              # Removed: 'unet/diffusion_pytorch_model.safetensors',
          ]
          self.assertFalse(is_safetensors_compatible(lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		[
              '''text_encoder/pytorch_model.bin''',
              '''text_encoder/model.safetensors''',
          ]
          self.assertTrue(is_safetensors_compatible(lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		[
              '''safety_checker/pytorch_model.bin''',
              '''safety_checker/model.safetensors''',
              '''vae/diffusion_pytorch_model.bin''',
              '''vae/diffusion_pytorch_model.safetensors''',
              '''text_encoder/pytorch_model.bin''',
              # Removed: 'text_encoder/model.safetensors',
              '''unet/diffusion_pytorch_model.bin''',
              '''unet/diffusion_pytorch_model.safetensors''',
          ]
          self.assertFalse(is_safetensors_compatible(lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  str    						=		[
              '''safety_checker/pytorch_model.fp16.bin''',
              '''safety_checker/model.fp16.safetensors''',
              '''vae/diffusion_pytorch_model.fp16.bin''',
              '''vae/diffusion_pytorch_model.fp16.safetensors''',
              '''text_encoder/pytorch_model.fp16.bin''',
              '''text_encoder/model.fp16.safetensors''',
              '''unet/diffusion_pytorch_model.fp16.bin''',
              '''unet/diffusion_pytorch_model.fp16.safetensors''',
          ]
          SCREAMING_SNAKE_CASE_      :  Any    						=		'''fp16'''
          self.assertTrue(is_safetensors_compatible(lowercase_						,	variant=lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		[
              '''unet/diffusion_pytorch_model.fp16.bin''',
              '''unet/diffusion_pytorch_model.fp16.safetensors''',
          ]
          SCREAMING_SNAKE_CASE_      :  Dict    						=		'''fp16'''
          self.assertTrue(is_safetensors_compatible(lowercase_						,	variant=lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		[
              '''unet/diffusion_pytorch_model.bin''',
              '''unet/diffusion_pytorch_model.safetensors''',
          ]
          SCREAMING_SNAKE_CASE_      :  Any    						=		'''fp16'''
          self.assertTrue(is_safetensors_compatible(lowercase_						,	variant=lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  Union[str, Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		[
              '''safety_checker/pytorch_model.fp16.bin''',
              '''safety_checker/model.fp16.safetensors''',
              '''vae/diffusion_pytorch_model.fp16.bin''',
              '''vae/diffusion_pytorch_model.fp16.safetensors''',
              '''text_encoder/pytorch_model.fp16.bin''',
              '''text_encoder/model.fp16.safetensors''',
              '''unet/diffusion_pytorch_model.fp16.bin''',
              # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
          ]
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		'''fp16'''
          self.assertFalse(is_safetensors_compatible(lowercase_						,	variant=lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		[
              '''text_encoder/pytorch_model.fp16.bin''',
              '''text_encoder/model.fp16.safetensors''',
          ]
          SCREAMING_SNAKE_CASE_      :  Any    						=		'''fp16'''
          self.assertTrue(is_safetensors_compatible(lowercase_						,	variant=lowercase_))
      def _SCREAMING_SNAKE_CASE (						self     :  List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		[
              '''text_encoder/pytorch_model.bin''',
              '''text_encoder/model.safetensors''',
          ]
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		'''fp16'''
          self.assertTrue(is_safetensors_compatible(lowercase_						,	variant=lowercase_))



      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		[
              '''safety_checker/pytorch_model.fp16.bin''',
              '''safety_checker/model.fp16.safetensors''',
              '''vae/diffusion_pytorch_model.fp16.bin''',
              '''vae/diffusion_pytorch_model.fp16.safetensors''',
              '''text_encoder/pytorch_model.fp16.bin''',
              # 'text_encoder/model.fp16.safetensors',
              '''unet/diffusion_pytorch_model.fp16.bin''',
              '''unet/diffusion_pytorch_model.fp16.safetensors''',
          ]
          SCREAMING_SNAKE_CASE_      :  str    						=		'''fp16'''
          self.assertFalse(is_safetensors_compatible(lowercase_						,	variant=lowercase_))


