


import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 						lowerCamelCase_							(					lowerCAmelCase__  ,  unittest.TestCase					):



					'''simple docstring'''







					__UpperCAmelCase							  =					CLIPTokenizer
					__UpperCAmelCase							  =					CLIPTokenizerFast
					__UpperCAmelCase							  =					True
					__UpperCAmelCase							  =					{}
					__UpperCAmelCase							  =					False






					def A   (					self    ) ->      Optional[Any]:







								'''simple docstring'''

								super().setUp()

								# fmt: off
								__lowercase			       = ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
								# fmt: on
								__lowercase			       = dict(zip(snake_case_			,							range(len(snake_case_    )    )    )    )
								__lowercase			       = ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
								__lowercase			       = {'''unk_token''': '''<unk>'''}

								__lowercase			       = os.path.join(self.tmpdirname			,							VOCAB_FILES_NAMES['''vocab_file''']    )
								__lowercase			       = os.path.join(self.tmpdirname			,							VOCAB_FILES_NAMES['''merges_file''']    )
								with open(self.vocab_file			,							'''w'''			,							encoding='''utf-8'''    ) as fp:
											fp.write(json.dumps(snake_case_    ) + '''\n'''    )
								with open(self.merges_file			,							'''w'''			,							encoding='''utf-8'''    ) as fp:
											fp.write('''\n'''.join(snake_case_    )    )






					def A   (					self			,							**snake_case_    ) ->      int:







								'''simple docstring'''

								kwargs.update(self.special_tokens_map    )
								return CLIPTokenizer.from_pretrained(self.tmpdirname			,							**snake_case_    )






					def A   (					self			,							**snake_case_    ) ->      str:







								'''simple docstring'''

								kwargs.update(self.special_tokens_map    )
								return CLIPTokenizerFast.from_pretrained(self.tmpdirname			,							**snake_case_    )






					def A   (					self			,							snake_case_    ) ->      Union[str, Any]:







								'''simple docstring'''

								__lowercase			       = '''lower newer'''
								__lowercase			       = '''lower newer'''
								return input_text, output_text






					def A   (					self    ) ->      Tuple:







								'''simple docstring'''

								__lowercase			       = CLIPTokenizer(self.vocab_file			,							self.merges_file			,							**self.special_tokens_map    )
								__lowercase			       = '''lower newer'''
								__lowercase			       = ['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
								__lowercase			       = tokenizer.tokenize(snake_case_    )
								self.assertListEqual(snake_case_			,							snake_case_    )

								__lowercase			       = tokens + [tokenizer.unk_token]
								__lowercase			       = [1_0, 2, 1_6, 9, 3, 2, 1_6, 2_0]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_    )			,							snake_case_    )






					@require_ftfy
					def A   (					self    ) ->      Optional[Any]:







								'''simple docstring'''

								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'    ):
														__lowercase			       = self.tokenizer_class.from_pretrained(snake_case_			,							**snake_case_    )
														__lowercase			       = self.rust_tokenizer_class.from_pretrained(snake_case_			,							**snake_case_    )

														__lowercase			       = '''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
														__lowercase			       = tokenizer_s.tokenize(snake_case_    )
														__lowercase			       = tokenizer_r.tokenize(snake_case_    )

														self.assertListEqual(snake_case_			,							snake_case_    )

														# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
														# with Tilde) encoded in 2 different ways
														__lowercase			       = '''xa\u0303y''' + ''' ''' + '''x\xe3y'''
														__lowercase			       = tokenizer_s.tokenize(snake_case_    )
														__lowercase			       = tokenizer_r.tokenize(snake_case_    )

														self.assertListEqual(snake_case_			,							snake_case_    )

														# Test that the tokenization is identical on unicode of space type
														__lowercase			       = [
														    '''\u0009''',  # (horizontal tab, '\t')
														    '''\u000B''',  # (vertical tab)
														    '''\u000C''',  # (form feed)
														    '''\u0020''',  # (space, ' ')
														    '''\u200E''',  # (left-to-right mark):w
														    '''\u200F''',  # (right-to-left mark)
														]
														for unicode_seq in spaces_unicodes:
																	__lowercase			       = tokenizer_s.tokenize(snake_case_    )
																	__lowercase			       = tokenizer_r.tokenize(snake_case_    )

																	self.assertListEqual(snake_case_			,							snake_case_    )

														# Test that the tokenization is identical on unicode of line break type
														__lowercase			       = [
														    '''\u000A''',  # (line feed, '\n')
														    '''\r\n''',  # (carriage return and line feed, '\r\n')
														    '''\u000D''',  # (carriage return, '\r')
														    '''\r''',  # (carriage return, '\r')
														    '''\u000D''',  # (carriage return, '\r')
														    '''\u2028''',  # (line separator)
														    '''\u2029''',  # (paragraph separator)
														    # "\u0085", # (next line)
														]

														# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
														# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
														# space (and thus into an empty list).

														for unicode_seq in line_break_unicodes:
																	__lowercase			       = tokenizer_s.tokenize(snake_case_    )
																	__lowercase			       = tokenizer_r.tokenize(snake_case_    )

																	self.assertListEqual(snake_case_			,							snake_case_    )






					def A   (					self    ) ->      Optional[Any]:







								'''simple docstring'''

								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'    ):
														__lowercase			       = '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
														__lowercase			       = F'{text_of_1_token} {text_of_1_token}'

														__lowercase			       = self.rust_tokenizer_class.from_pretrained(
														    snake_case_			,							use_fast=snake_case_			,							)
														__lowercase			       = tokenizer_r(snake_case_			,							return_offsets_mapping=snake_case_			,							add_special_tokens=snake_case_    )
														self.assertEqual(encoding.offset_mapping[0]			,							(0, len(snake_case_    ))    )
														self.assertEqual(
														    encoding.offset_mapping[1]			,							(len(snake_case_    ) + 1, len(snake_case_    ) + 1 + len(snake_case_    ))			,							)

														__lowercase			       = F' {text}'

														__lowercase			       = self.rust_tokenizer_class.from_pretrained(
														    snake_case_			,							use_fast=snake_case_			,							)
														__lowercase			       = tokenizer_r(snake_case_			,							return_offsets_mapping=snake_case_			,							add_special_tokens=snake_case_    )
														self.assertEqual(encoding.offset_mapping[0]			,							(1, 1 + len(snake_case_    ))    )
														self.assertEqual(
														    encoding.offset_mapping[1]			,							(1 + len(snake_case_    ) + 1, 1 + len(snake_case_    ) + 1 + len(snake_case_    ))			,							)






					def A   (					self    ) ->      List[Any]:







								'''simple docstring'''

								with self.assertRaises(snake_case_    ) as context:
											self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''    )

								self.assertTrue(
								    context.exception.args[0].startswith(
								        '''The `backend_tokenizer` provided does not match the expected format.'''    )    )






					@require_ftfy
					def A   (					self    ) ->      Dict:







								'''simple docstring'''

								super().test_tokenization_python_rust_equals()






					def A   (					self    ) ->      Union[str, Any]:







								'''simple docstring'''

								pass






from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
	import PIL


a			:			List[str]												=					logging.get_logger(__name__)
class 						lowerCamelCase_							(					lowerCAmelCase__					):



					'''simple docstring'''







					__UpperCAmelCase							  =					["pixel_values"]






					def __init__(					self			,							snake_case_ = True			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = PILImageResampling.BILINEAR			,							snake_case_ = True			,							snake_case_ = 1 / 2_5_5			,							snake_case_ = True			,							snake_case_ = None			,							snake_case_ = None			,							**snake_case_			,							) ->      None:







								'''simple docstring'''

								super().__init__(**snake_case_    )
								__lowercase			       = size if size is not None else {'''shortest_edge''': 3_8_4}
								__lowercase			       = get_size_dict(snake_case_			,							default_to_square=snake_case_    )

								__lowercase			       = do_resize
								__lowercase			       = size
								# Default value set here for backwards compatibility where the value in config is None
								__lowercase			       = crop_pct if crop_pct is not None else 2_2_4 / 2_5_6
								__lowercase			       = resample
								__lowercase			       = do_rescale
								__lowercase			       = rescale_factor
								__lowercase			       = do_normalize
								__lowercase			       = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
								__lowercase			       = image_std if image_std is not None else IMAGENET_STANDARD_STD






					def A   (					self			,							snake_case_			,							snake_case_			,							snake_case_			,							snake_case_ = PILImageResampling.BICUBIC			,							snake_case_ = None			,							**snake_case_			,							) ->      np.ndarray:







								'''simple docstring'''

								__lowercase			       = get_size_dict(snake_case_			,							default_to_square=snake_case_    )
								if "shortest_edge" not in size:
											raise ValueError(F'Size dictionary must contain \'shortest_edge\' key. Got {size.keys()}'    )
								__lowercase			       = size['''shortest_edge''']

								if shortest_edge < 3_8_4:
											# maintain same ratio, resizing shortest edge to shortest_edge/crop_pct
											__lowercase			       = int(shortest_edge / crop_pct    )
											__lowercase			       = get_resize_output_image_size(snake_case_			,							size=snake_case_			,							default_to_square=snake_case_    )
											__lowercase			       = resize(image=snake_case_			,							size=snake_case_			,							resample=snake_case_			,							data_format=snake_case_			,							**snake_case_    )
											# then crop to (shortest_edge, shortest_edge)
											return center_crop(image=snake_case_			,							size=(shortest_edge, shortest_edge)			,							data_format=snake_case_			,							**snake_case_    )
								else:
											# warping (no cropping) when evaluated at 384 or larger
											return resize(
											    snake_case_			,							size=(shortest_edge, shortest_edge)			,							resample=snake_case_			,							data_format=snake_case_			,							**snake_case_    )






					def A   (					self			,							snake_case_			,							snake_case_			,							snake_case_ = None			,							**snake_case_			,							) ->      Optional[int]:







								'''simple docstring'''

								return rescale(snake_case_			,							scale=snake_case_			,							data_format=snake_case_			,							**snake_case_    )






					def A   (					self			,							snake_case_			,							snake_case_			,							snake_case_			,							snake_case_ = None			,							**snake_case_			,							) ->      np.ndarray:







								'''simple docstring'''

								return normalize(snake_case_			,							mean=snake_case_			,							std=snake_case_			,							data_format=snake_case_			,							**snake_case_    )






					def A   (					self			,							snake_case_			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = ChannelDimension.FIRST			,							**snake_case_			,							) ->      PIL.Image.Image:







								'''simple docstring'''

								__lowercase			       = do_resize if do_resize is not None else self.do_resize
								__lowercase			       = crop_pct if crop_pct is not None else self.crop_pct
								__lowercase			       = resample if resample is not None else self.resample
								__lowercase			       = do_rescale if do_rescale is not None else self.do_rescale
								__lowercase			       = rescale_factor if rescale_factor is not None else self.rescale_factor
								__lowercase			       = do_normalize if do_normalize is not None else self.do_normalize
								__lowercase			       = image_mean if image_mean is not None else self.image_mean
								__lowercase			       = image_std if image_std is not None else self.image_std

								__lowercase			       = size if size is not None else self.size
								__lowercase			       = get_size_dict(snake_case_			,							default_to_square=snake_case_    )

								__lowercase			       = make_list_of_images(snake_case_    )

								if not valid_images(snake_case_    ):
											raise ValueError(
											    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
											    '''torch.Tensor, tf.Tensor or jax.ndarray.'''    )

								if do_resize and size is None or resample is None:
											raise ValueError('''Size and resample must be specified if do_resize is True.'''    )

								if do_resize and size["shortest_edge"] < 3_8_4 and crop_pct is None:
											raise ValueError('''crop_pct must be specified if size < 384.'''    )

								if do_rescale and rescale_factor is None:
											raise ValueError('''Rescale factor must be specified if do_rescale is True.'''    )

								if do_normalize and (image_mean is None or image_std is None):
											raise ValueError('''Image mean and std must be specified if do_normalize is True.'''    )

								# All transformations expect numpy arrays.
								__lowercase			       = [to_numpy_array(snake_case_    ) for image in images]

								if do_resize:
											__lowercase			       = [self.resize(image=snake_case_			,							size=snake_case_			,							crop_pct=snake_case_			,							resample=snake_case_    ) for image in images]

								if do_rescale:
											__lowercase			       = [self.rescale(image=snake_case_			,							scale=snake_case_    ) for image in images]

								if do_normalize:
											__lowercase			       = [self.normalize(image=snake_case_			,							mean=snake_case_			,							std=snake_case_    ) for image in images]

								__lowercase			       = [to_channel_dimension_format(snake_case_			,							snake_case_    ) for image in images]

								__lowercase			       = {'''pixel_values''': images}
								return BatchFeature(data=snake_case_			,							tensor_type=snake_case_    )


