

"""simple docstring"""


from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
  import PIL

a    :int			=       logging.get_logger(__name__)




def        _lowercase    (			__lowerCAmelCase							)				-> List[List[ImageInput]]:






      if isinstance(__a     ,    (list, tuple)							) and isinstance(videos[0]     ,    (list, tuple)							) and is_valid_image(videos[0][0]							):
            return videos

      elif isinstance(__a     ,    (list, tuple)							) and is_valid_image(videos[0]							):
            return [videos]

      elif is_valid_image(__a							):
            return [[videos]]

      raise ValueError(F'''Could not make batched video from {videos}'''							)



class    __a							(UpperCAmelCase__):






     '''simple docstring'''
     _SCREAMING_SNAKE_CASE  :Tuple				=       ["""pixel_values"""]

     def __init__(  self		,			_a = True		,			_a = None		,			_a = PILImageResampling.BILINEAR		,			_a = True		,			_a = None		,			_a = True		,			_a = 1 / 255		,			_a = True		,			_a = True		,			_a = None		,			_a = None		,			**_a		,			)     ->					Optional[Any]:


           """simple docstring"""


           super().__init__(**lowercase_     )
           SCREAMING_SNAKE_CASE__      :					str											=		size if size is not None else {'''shortest_edge''': 256}
           SCREAMING_SNAKE_CASE__      :					Optional[int]											=		get_size_dict(lowercase_		,			default_to_square=lowercase_     )
           SCREAMING_SNAKE_CASE__      :					Union[str, Any]											=		crop_size if crop_size is not None else {'''height''': 224, '''width''': 224}
           SCREAMING_SNAKE_CASE__      :					Dict											=		get_size_dict(lowercase_		,			param_name="""crop_size"""     )

           SCREAMING_SNAKE_CASE__      :					Optional[int]											=		do_resize
           SCREAMING_SNAKE_CASE__      :					List[Any]											=		size
           SCREAMING_SNAKE_CASE__      :					Tuple											=		do_center_crop
           SCREAMING_SNAKE_CASE__      :					Dict											=		crop_size
           SCREAMING_SNAKE_CASE__      :					List[Any]											=		resample
           SCREAMING_SNAKE_CASE__      :					List[str]											=		do_rescale
           SCREAMING_SNAKE_CASE__      :					List[str]											=		rescale_factor
           SCREAMING_SNAKE_CASE__      :					List[Any]											=		offset
           SCREAMING_SNAKE_CASE__      :					List[Any]											=		do_normalize
           SCREAMING_SNAKE_CASE__      :					Tuple											=		image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
           SCREAMING_SNAKE_CASE__      :					Optional[int]											=		image_std if image_std is not None else IMAGENET_STANDARD_STD

     def     _a	(  self		,			_a		,			_a		,			_a = PILImageResampling.BILINEAR		,			_a = None		,			**_a		,			)     ->					Dict:


           """simple docstring"""


           SCREAMING_SNAKE_CASE__      :					Tuple											=		get_size_dict(lowercase_		,			default_to_square=lowercase_     )
           if "shortest_edge" in size:
                 SCREAMING_SNAKE_CASE__      :					List[Any]											=		get_resize_output_image_size(lowercase_		,			size["""shortest_edge"""]		,			default_to_square=lowercase_     )
           elif "height" in size and "width" in size:
                 SCREAMING_SNAKE_CASE__      :					Union[str, Any]											=		(size['''height'''], size['''width'''])
           else:
                 raise ValueError(f'''Size must have \'height\' and \'width\' or \'shortest_edge\' as keys. Got {size.keys()}'''     )
           return resize(lowercase_		,			size=lowercase_		,			resample=lowercase_		,			data_format=lowercase_		,			**lowercase_     )

     def     _a	(  self		,			_a		,			_a		,			_a = None		,			**_a		,			)     ->					Optional[int]:


           """simple docstring"""


           SCREAMING_SNAKE_CASE__      :					List[str]											=		get_size_dict(lowercase_     )
           if "height" not in size or "width" not in size:
                 raise ValueError(f'''Size must have \'height\' and \'width\' as keys. Got {size.keys()}'''     )
           return center_crop(lowercase_		,			size=(size["""height"""], size["""width"""])		,			data_format=lowercase_		,			**lowercase_     )

     def     _a	(  self		,			_a		,			_a		,			_a = True		,			_a = None		,			**_a		,			)     ->					Dict:


           """simple docstring"""


           SCREAMING_SNAKE_CASE__      :					List[Any]											=		image.astype(np.floataa     )
           if offset:
                 SCREAMING_SNAKE_CASE__      :					Tuple											=		image - (scale / 2)
           return rescale(lowercase_		,			scale=lowercase_		,			data_format=lowercase_		,			**lowercase_     )

     def     _a	(  self		,			_a		,			_a		,			_a		,			_a = None		,			**_a		,			)     ->					List[Any]:


           """simple docstring"""


           return normalize(lowercase_		,			mean=lowercase_		,			std=lowercase_		,			data_format=lowercase_		,			**lowercase_     )

     def     _a	(  self		,			_a		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = ChannelDimension.FIRST		,			)     ->					List[str]:


           """simple docstring"""


           if do_resize and size is None or resample is None:
                 raise ValueError("""Size and resample must be specified if do_resize is True."""     )

           if do_center_crop and crop_size is None:
                 raise ValueError("""Crop size must be specified if do_center_crop is True."""     )

           if do_rescale and rescale_factor is None:
                 raise ValueError("""Rescale factor must be specified if do_rescale is True."""     )

           if do_normalize and (image_mean is None or image_std is None):
                 raise ValueError("""Image mean and std must be specified if do_normalize is True."""     )

           if offset and not do_rescale:
                 raise ValueError("""For offset, do_rescale must also be set to True."""     )

           # All transformations expect numpy arrays.
           SCREAMING_SNAKE_CASE__      :					List[str]											=		to_numpy_array(lowercase_     )

           if do_resize:
                 SCREAMING_SNAKE_CASE__      :					List[Any]											=		self.resize(image=lowercase_		,			size=lowercase_		,			resample=lowercase_     )

           if do_center_crop:
                 SCREAMING_SNAKE_CASE__      :					Dict											=		self.center_crop(lowercase_		,			size=lowercase_     )

           if do_rescale:
                 SCREAMING_SNAKE_CASE__      :					int											=		self.rescale(image=lowercase_		,			scale=lowercase_		,			offset=lowercase_     )

           if do_normalize:
                 SCREAMING_SNAKE_CASE__      :					Dict											=		self.normalize(image=lowercase_		,			mean=lowercase_		,			std=lowercase_     )

           SCREAMING_SNAKE_CASE__      :					List[Any]											=		to_channel_dimension_format(lowercase_		,			lowercase_     )
           return image





     def     _a	(  self		,			_a		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = None		,			_a = ChannelDimension.FIRST		,			**_a		,			)     ->					List[str]:


           """simple docstring"""


           SCREAMING_SNAKE_CASE__      :					Optional[Any]											=		do_resize if do_resize is not None else self.do_resize
           SCREAMING_SNAKE_CASE__      :					int											=		resample if resample is not None else self.resample
           SCREAMING_SNAKE_CASE__      :					Union[str, Any]											=		do_center_crop if do_center_crop is not None else self.do_center_crop
           SCREAMING_SNAKE_CASE__      :					Dict											=		do_rescale if do_rescale is not None else self.do_rescale
           SCREAMING_SNAKE_CASE__      :					Dict											=		rescale_factor if rescale_factor is not None else self.rescale_factor
           SCREAMING_SNAKE_CASE__      :					Dict											=		offset if offset is not None else self.offset
           SCREAMING_SNAKE_CASE__      :					str											=		do_normalize if do_normalize is not None else self.do_normalize
           SCREAMING_SNAKE_CASE__      :					Dict											=		image_mean if image_mean is not None else self.image_mean
           SCREAMING_SNAKE_CASE__      :					List[str]											=		image_std if image_std is not None else self.image_std

           SCREAMING_SNAKE_CASE__      :					Union[str, Any]											=		size if size is not None else self.size
           SCREAMING_SNAKE_CASE__      :					Tuple											=		get_size_dict(lowercase_		,			default_to_square=lowercase_     )
           SCREAMING_SNAKE_CASE__      :					Any											=		crop_size if crop_size is not None else self.crop_size
           SCREAMING_SNAKE_CASE__      :					Dict											=		get_size_dict(lowercase_		,			param_name="""crop_size"""     )

           if not valid_images(lowercase_     ):
                 raise ValueError(
                     """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
                     """torch.Tensor, tf.Tensor or jax.ndarray."""     )

           SCREAMING_SNAKE_CASE__      :					Tuple											=		make_batched(lowercase_     )

           SCREAMING_SNAKE_CASE__      :					Optional[Any]											=		[
               [
                   self._preprocess_image(
                       image=lowercase_		,			do_resize=lowercase_		,			size=lowercase_		,			resample=lowercase_		,			do_center_crop=lowercase_		,			crop_size=lowercase_		,			do_rescale=lowercase_		,			rescale_factor=lowercase_		,			offset=lowercase_		,			do_normalize=lowercase_		,			image_mean=lowercase_		,			image_std=lowercase_		,			data_format=lowercase_		,			)
                   for img in video
               ]
               for video in videos
           ]

           SCREAMING_SNAKE_CASE__      :					int											=		{'''pixel_values''': videos}
           return BatchFeature(data=lowercase_		,			tensor_type=lowercase_     )












"""simple docstring"""






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


UpperCAmelCase_   : Optional[int]   			= logging.get_logger(__name__)

UpperCAmelCase_   : Dict   			= {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase_   : Dict   			= {
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

UpperCAmelCase_   : List[str]   			= {
    """gpt2""": 1024,
    """gpt2-medium""": 1024,
    """gpt2-large""": 1024,
    """gpt2-xl""": 1024,
    """distilgpt2""": 1024,
}




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      __UpperCamelCase	 =  VOCAB_FILES_NAMES
      __UpperCamelCase	 =  PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase	 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase	 =  ["input_ids", "attention_mask"]
      __UpperCamelCase	 =  GPTaTokenizer
      def __init__(						self     :  Optional[int]						,	lowercase_     :  int=None						,	lowercase_     :  List[str]=None						,	lowercase_     :  Union[str, Any]=None						,	lowercase_     :  Tuple="<|endoftext|>"						,	lowercase_     :  str="<|endoftext|>"						,	lowercase_     :  Dict="<|endoftext|>"						,	lowercase_     :  Tuple=False						,	**lowercase_     :  Optional[int]						,	):




          '''simple docstring'''


          super().__init__(
              lowercase_						,	lowercase_						,	tokenizer_file=lowercase_						,	unk_token=lowercase_						,	bos_token=lowercase_						,	eos_token=lowercase_						,	add_prefix_space=lowercase_						,	**lowercase_						,	)

          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		kwargs.pop('''add_bos_token'''						,	lowercase_)

          SCREAMING_SNAKE_CASE_      :  Dict    						=		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
          if pre_tok_state.get('''add_prefix_space'''						,	lowercase_) != add_prefix_space:
              SCREAMING_SNAKE_CASE_      :  int    						=		getattr(lowercase_						,	pre_tok_state.pop('''type'''))
              SCREAMING_SNAKE_CASE_      :  str    						=		add_prefix_space
              SCREAMING_SNAKE_CASE_      :  Dict    						=		pre_tok_class(**lowercase_)

          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		add_prefix_space
      def _SCREAMING_SNAKE_CASE (						self     :  str						,	*lowercase_     :  List[Any]						,	**lowercase_     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		kwargs.get('''is_split_into_words'''						,	lowercase_)
          assert self.add_prefix_space or not is_split_into_words, (
              F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
              "to use it with pretokenized inputs."
          )

          return super()._batch_encode_plus(*lowercase_						,	**lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]						,	*lowercase_     :  List[str]						,	**lowercase_     :  List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		kwargs.get('''is_split_into_words'''						,	lowercase_)

          assert self.add_prefix_space or not is_split_into_words, (
              F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
              "to use it with pretokenized inputs."
          )

          return super()._encode_plus(*lowercase_						,	**lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  str						,	lowercase_     :  Optional[str] = None):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		self._tokenizer.model.save(lowercase_						,	name=lowercase_)
          return tuple(lowercase_)



      def _SCREAMING_SNAKE_CASE (						self     :  int						,	lowercase_     :  "Conversation"):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Dict    						=		[]
          for is_user, text in conversation.iter_texts():
              input_ids.extend(self.encode(lowercase_						,	add_special_tokens=lowercase_) + [self.eos_token_id])

          if len(lowercase_) > self.model_max_length:
              SCREAMING_SNAKE_CASE_      :  Any    						=		input_ids[-self.model_max_length :]
          return input_ids


