



"""simple docstring"""






from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import convert_to_rgb, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
	import PIL


__snake_case							:   Dict      =    logging.get_logger(__name__)



class 			A__   (  __SCREAMING_SNAKE_CASE							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE          =	['pixel_values']

	def __init__(     self:		Tuple       ,			_SCREAMING_SNAKE_CASE:		bool = True       ,			_SCREAMING_SNAKE_CASE:		Dict[str, int] = None       ,			_SCREAMING_SNAKE_CASE:		PILImageResampling = PILImageResampling.BICUBIC       ,			_SCREAMING_SNAKE_CASE:		bool = True       ,			_SCREAMING_SNAKE_CASE:		Union[int, float] = 1 / 255       ,			_SCREAMING_SNAKE_CASE:		bool = True       ,			_SCREAMING_SNAKE_CASE:		Optional[Union[float, List[float]]] = None       ,			_SCREAMING_SNAKE_CASE:		Optional[Union[float, List[float]]] = None       ,			_SCREAMING_SNAKE_CASE:		bool = True       ,			**_SCREAMING_SNAKE_CASE:		int       ,			)				->       None:

				"""simple docstring"""

				super().__init__(**_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase      :  List[str]												=       size if size is not None else {"height": 384, "width": 384}
				__lowerCAmelCase      :  Union[str, Any]												=       get_size_dict(_SCREAMING_SNAKE_CASE       ,			default_to_square=_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase      :  int												=       do_resize
				__lowerCAmelCase      :  Optional[Any]												=       size
				__lowerCAmelCase      :  Any												=       resample
				__lowerCAmelCase      :  Union[str, Any]												=       do_rescale
				__lowerCAmelCase      :  Optional[int]												=       rescale_factor
				__lowerCAmelCase      :  Dict												=       do_normalize
				__lowerCAmelCase      :  Any												=       image_mean if image_mean is not None else OPENAI_CLIP_MEAN
				__lowerCAmelCase      :  Tuple												=       image_std if image_std is not None else OPENAI_CLIP_STD
				__lowerCAmelCase      :  List[Any]												=       do_convert_rgb

	def 							_SCREAMING_SNAKE_CASE   (     self:		Tuple       ,			_SCREAMING_SNAKE_CASE:		np.ndarray       ,			_SCREAMING_SNAKE_CASE:		Dict[str, int]       ,			_SCREAMING_SNAKE_CASE:		PILImageResampling = PILImageResampling.BICUBIC       ,			_SCREAMING_SNAKE_CASE:		Optional[Union[str, ChannelDimension]] = None       ,			**_SCREAMING_SNAKE_CASE:		int       ,			)				->       np.ndarray:

				"""simple docstring"""

				__lowerCAmelCase      :  int												=       get_size_dict(_SCREAMING_SNAKE_CASE       ,			default_to_square=_SCREAMING_SNAKE_CASE)
				if "height" not in size or "width" not in size:
							raise ValueError(F"""The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}""")
				__lowerCAmelCase      :  Tuple												=       (size["height"], size["width"])
				return resize(_SCREAMING_SNAKE_CASE       ,			size=_SCREAMING_SNAKE_CASE       ,			resample=_SCREAMING_SNAKE_CASE       ,			data_format=_SCREAMING_SNAKE_CASE       ,			**_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Dict       ,			_SCREAMING_SNAKE_CASE:		np.ndarray       ,			_SCREAMING_SNAKE_CASE:		Union[int, float]       ,			_SCREAMING_SNAKE_CASE:		Optional[Union[str, ChannelDimension]] = None       ,			**_SCREAMING_SNAKE_CASE:		List[str]       ,			)				->       List[Any]:

				"""simple docstring"""

				return rescale(_SCREAMING_SNAKE_CASE       ,			scale=_SCREAMING_SNAKE_CASE       ,			data_format=_SCREAMING_SNAKE_CASE       ,			**_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Any       ,			_SCREAMING_SNAKE_CASE:		np.ndarray       ,			_SCREAMING_SNAKE_CASE:		Union[float, List[float]]       ,			_SCREAMING_SNAKE_CASE:		Union[float, List[float]]       ,			_SCREAMING_SNAKE_CASE:		Optional[Union[str, ChannelDimension]] = None       ,			**_SCREAMING_SNAKE_CASE:		int       ,			)				->       np.ndarray:

				"""simple docstring"""

				return normalize(_SCREAMING_SNAKE_CASE       ,			mean=_SCREAMING_SNAKE_CASE       ,			std=_SCREAMING_SNAKE_CASE       ,			data_format=_SCREAMING_SNAKE_CASE       ,			**_SCREAMING_SNAKE_CASE)







	def 							_SCREAMING_SNAKE_CASE   (     self:		List[str]       ,			_SCREAMING_SNAKE_CASE:		ImageInput       ,			_SCREAMING_SNAKE_CASE:		Optional[bool] = None       ,			_SCREAMING_SNAKE_CASE:		Optional[Dict[str, int]] = None       ,			_SCREAMING_SNAKE_CASE:		PILImageResampling = None       ,			_SCREAMING_SNAKE_CASE:		Optional[bool] = None       ,			_SCREAMING_SNAKE_CASE:		Optional[float] = None       ,			_SCREAMING_SNAKE_CASE:		Optional[bool] = None       ,			_SCREAMING_SNAKE_CASE:		Optional[Union[float, List[float]]] = None       ,			_SCREAMING_SNAKE_CASE:		Optional[Union[float, List[float]]] = None       ,			_SCREAMING_SNAKE_CASE:		Optional[Union[str, TensorType]] = None       ,			_SCREAMING_SNAKE_CASE:		bool = None       ,			_SCREAMING_SNAKE_CASE:		ChannelDimension = ChannelDimension.FIRST       ,			**_SCREAMING_SNAKE_CASE:		int       ,			)				->       PIL.Image.Image:

				"""simple docstring"""

				__lowerCAmelCase      :  Optional[Any]												=       do_resize if do_resize is not None else self.do_resize
				__lowerCAmelCase      :  Union[str, Any]												=       resample if resample is not None else self.resample
				__lowerCAmelCase      :  Optional[int]												=       do_rescale if do_rescale is not None else self.do_rescale
				__lowerCAmelCase      :  Optional[Any]												=       rescale_factor if rescale_factor is not None else self.rescale_factor
				__lowerCAmelCase      :  str												=       do_normalize if do_normalize is not None else self.do_normalize
				__lowerCAmelCase      :  Optional[int]												=       image_mean if image_mean is not None else self.image_mean
				__lowerCAmelCase      :  Optional[Any]												=       image_std if image_std is not None else self.image_std
				__lowerCAmelCase      :  Optional[int]												=       do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

				__lowerCAmelCase      :  str												=       size if size is not None else self.size
				__lowerCAmelCase      :  List[str]												=       get_size_dict(_SCREAMING_SNAKE_CASE       ,			default_to_square=_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase      :  Any												=       make_list_of_images(_SCREAMING_SNAKE_CASE)

				if not valid_images(_SCREAMING_SNAKE_CASE):
							raise ValueError(
							    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
							    "torch.Tensor, tf.Tensor or jax.ndarray.")

				if do_resize and size is None or resample is None:
							raise ValueError("Size and resample must be specified if do_resize is True.")

				if do_rescale and rescale_factor is None:
							raise ValueError("Rescale factor must be specified if do_rescale is True.")

				if do_normalize and (image_mean is None or image_std is None):
							raise ValueError("Image mean and std must be specified if do_normalize is True.")

				# PIL RGBA images are converted to RGB
				if do_convert_rgb:
							__lowerCAmelCase      :  List[Any]												=       [convert_to_rgb(_SCREAMING_SNAKE_CASE) for image in images]

				# All transformations expect numpy arrays.
				__lowerCAmelCase      :  Any												=       [to_numpy_array(_SCREAMING_SNAKE_CASE) for image in images]

				if do_resize:
							__lowerCAmelCase      :  str												=       [self.resize(image=_SCREAMING_SNAKE_CASE       ,			size=_SCREAMING_SNAKE_CASE       ,			resample=_SCREAMING_SNAKE_CASE) for image in images]

				if do_rescale:
							__lowerCAmelCase      :  List[Any]												=       [self.rescale(image=_SCREAMING_SNAKE_CASE       ,			scale=_SCREAMING_SNAKE_CASE) for image in images]

				if do_normalize:
							__lowerCAmelCase      :  List[str]												=       [self.normalize(image=_SCREAMING_SNAKE_CASE       ,			mean=_SCREAMING_SNAKE_CASE       ,			std=_SCREAMING_SNAKE_CASE) for image in images]

				__lowerCAmelCase      :  Any												=       [to_channel_dimension_format(_SCREAMING_SNAKE_CASE       ,			_SCREAMING_SNAKE_CASE) for image in images]

				__lowerCAmelCase      :  Optional[Any]												=       BatchFeature(data={"pixel_values": images}       ,			tensor_type=_SCREAMING_SNAKE_CASE)

				return encoded_outputs




"""simple docstring"""






import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 			A__   (  __SCREAMING_SNAKE_CASE  ,		unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE          =	FunnelTokenizer
	SCREAMING_SNAKE_CASE          =	FunnelTokenizerFast
	SCREAMING_SNAKE_CASE          =	True
	SCREAMING_SNAKE_CASE          =	True

	def 							_SCREAMING_SNAKE_CASE   (     self:		Tuple)				->       Optional[int]:

				"""simple docstring"""

				super().setUp()

				__lowerCAmelCase      :  str												=       [
				    "<unk>",
				    "<cls>",
				    "<sep>",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				__lowerCAmelCase      :  int												=       os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["vocab_file"])
				with open(self.vocab_file       ,			"w"       ,			encoding="utf-8") as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))

	def 							_SCREAMING_SNAKE_CASE   (     self:		Optional[Any]       ,			**_SCREAMING_SNAKE_CASE:		Union[str, Any])				->       int:

				"""simple docstring"""

				return FunnelTokenizer.from_pretrained(self.tmpdirname       ,			**_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Any       ,			**_SCREAMING_SNAKE_CASE:		Any)				->       str:

				"""simple docstring"""

				return FunnelTokenizerFast.from_pretrained(self.tmpdirname       ,			**_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE   (     self:		Tuple       ,			_SCREAMING_SNAKE_CASE:		str)				->       Any:

				"""simple docstring"""

				__lowerCAmelCase      :  Union[str, Any]												=       "UNwant\u00E9d,running"
				__lowerCAmelCase      :  str												=       "unwanted, running"
				return input_text, output_text

	def 							_SCREAMING_SNAKE_CASE   (     self:		Optional[int])				->       List[str]:

				"""simple docstring"""

				__lowerCAmelCase      :  Any												=       self.tokenizer_class(self.vocab_file)

				__lowerCAmelCase      :  Any												=       tokenizer.tokenize("UNwant\u00E9d,running")
				self.assertListEqual(_SCREAMING_SNAKE_CASE       ,			["un", "##want", "##ed", ",", "runn", "##ing"])
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE)       ,			[7, 4, 5, 10, 8, 9])







	def 							_SCREAMING_SNAKE_CASE   (     self:		List[str])				->       List[str]:

				"""simple docstring"""

				__lowerCAmelCase      :  Optional[Any]												=       self.get_tokenizers(do_lower_case=_SCREAMING_SNAKE_CASE)
				for tokenizer in tokenizers:
							__lowerCAmelCase      :  List[str]												=       tokenizer("UNwant\u00E9d,running")
							__lowerCAmelCase      :  Optional[int]												=       len(inputs["input_ids"]) - 1
							self.assertListEqual(inputs["token_type_ids"]       ,			[2] + [0] * sentence_len)

							__lowerCAmelCase      :  List[str]												=       tokenizer("UNwant\u00E9d,running"       ,			"UNwant\u00E9d,running")
							self.assertListEqual(inputs["token_type_ids"]       ,			[2] + [0] * sentence_len + [1] * sentence_len)