

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class  lowercase				(       lowercase_	,  unittest.TestCase			):
				__SCREAMING_SNAKE_CASE					: Any       			=			CLIPTokenizer
				__SCREAMING_SNAKE_CASE					: Optional[Any]       			=			CLIPTokenizerFast
				__SCREAMING_SNAKE_CASE					: Union[str, Any]       			=			True
				__SCREAMING_SNAKE_CASE					: List[Any]       			=			{}
				__SCREAMING_SNAKE_CASE					: Tuple       			=			False




				def     a							(					self       ):
											super().setUp()

											# fmt: off
											snake_case_							=    ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
											# fmt: on
											snake_case_							=    dict(zip(__A      ,   range(len(__A       )       )       )       )
											snake_case_							=    ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
											snake_case_							=    {"unk_token": "<unk>"}

											snake_case_							=    os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['vocab_file']       )
											snake_case_							=    os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['merges_file']       )
											with open(self.vocab_file      ,   'w'      ,   encoding='utf-8'       ) as fp:
																		fp.write(json.dumps(__A       ) + '\n'       )
											with open(self.merges_file      ,   'w'      ,   encoding='utf-8'       ) as fp:
																		fp.write('\n'.join(__A       )       )




				def     a							(					self      ,   **snake_case       ):
											kwargs.update(self.special_tokens_map       )
											return CLIPTokenizer.from_pretrained(self.tmpdirname      ,   **__A       )




				def     a							(					self      ,   **snake_case       ):
											kwargs.update(self.special_tokens_map       )
											return CLIPTokenizerFast.from_pretrained(self.tmpdirname      ,   **__A       )




				def     a							(					self      ,   snake_case       ):
											snake_case_							=    "lower newer"
											snake_case_							=    "lower newer"
											return input_text, output_text




				def     a							(					self       ):
											snake_case_							=    CLIPTokenizer(self.vocab_file      ,   self.merges_file      ,   **self.special_tokens_map       )
											snake_case_							=    "lower newer"
											snake_case_							=    ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
											snake_case_							=    tokenizer.tokenize(__A       )
											self.assertListEqual(__A      ,   __A       )

											snake_case_							=    tokens + [tokenizer.unk_token]
											snake_case_							=    [10, 2, 16, 9, 3, 2, 16, 20]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(__A       )      ,   __A       )




				@require_ftfy
				def     a							(					self       ):





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																									snake_case_							=    self.tokenizer_class.from_pretrained(__A      ,   **__A       )
																									snake_case_							=    self.rust_tokenizer_class.from_pretrained(__A      ,   **__A       )

																									snake_case_							=    "A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
																									snake_case_							=    tokenizer_s.tokenize(__A       )
																									snake_case_							=    tokenizer_r.tokenize(__A       )

																									self.assertListEqual(__A      ,   __A       )

																									# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																									# with Tilde) encoded in 2 different ways
																									snake_case_							=    "xa\u0303y" + " " + "x\xe3y"
																									snake_case_							=    tokenizer_s.tokenize(__A       )
																									snake_case_							=    tokenizer_r.tokenize(__A       )

																									self.assertListEqual(__A      ,   __A       )

																									# Test that the tokenization is identical on unicode of space type
																									snake_case_							=    [
																									    "\u0009",  # (horizontal tab, '\t')
																									    "\u000B",  # (vertical tab)
																									    "\u000C",  # (form feed)
																									    "\u0020",  # (space, ' ')
																									    "\u200E",  # (left-to-right mark):w
																									    "\u200F",  # (right-to-left mark)
																									]
																									for unicode_seq in spaces_unicodes:
																																snake_case_							=    tokenizer_s.tokenize(__A       )
																																snake_case_							=    tokenizer_r.tokenize(__A       )

																																self.assertListEqual(__A      ,   __A       )

																									# Test that the tokenization is identical on unicode of line break type
																									snake_case_							=    [
																									    "\u000A",  # (line feed, '\n')
																									    "\r\n",  # (carriage return and line feed, '\r\n')
																									    "\u000D",  # (carriage return, '\r')
																									    "\r",  # (carriage return, '\r')
																									    "\u000D",  # (carriage return, '\r')
																									    "\u2028",  # (line separator)
																									    "\u2029",  # (paragraph separator)
																									    # "\u0085", # (next line)
																									]

																									# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																									# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																									# space (and thus into an empty list).

																									for unicode_seq in line_break_unicodes:
																																snake_case_							=    tokenizer_s.tokenize(__A       )
																																snake_case_							=    tokenizer_r.tokenize(__A       )

																																self.assertListEqual(__A      ,   __A       )




				def     a							(					self       ):





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																									snake_case_							=    "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
																									snake_case_							=    F'''{text_of_1_token} {text_of_1_token}'''

																									snake_case_							=    self.rust_tokenizer_class.from_pretrained(
																									    __A      ,   use_fast=__A      ,   )
																									snake_case_							=    tokenizer_r(__A      ,   return_offsets_mapping=__A      ,   add_special_tokens=__A       )
																									self.assertEqual(encoding.offset_mapping[0]      ,   (0, len(__A       ))       )
																									self.assertEqual(
																									    encoding.offset_mapping[1]      ,   (len(__A       ) + 1, len(__A       ) + 1 + len(__A       ))      ,   )

																									snake_case_							=    F''' {text}'''

																									snake_case_							=    self.rust_tokenizer_class.from_pretrained(
																									    __A      ,   use_fast=__A      ,   )
																									snake_case_							=    tokenizer_r(__A      ,   return_offsets_mapping=__A      ,   add_special_tokens=__A       )
																									self.assertEqual(encoding.offset_mapping[0]      ,   (1, 1 + len(__A       ))       )
																									self.assertEqual(
																									    encoding.offset_mapping[1]      ,   (1 + len(__A       ) + 1, 1 + len(__A       ) + 1 + len(__A       ))      ,   )




				def     a							(					self       ):





											with self.assertRaises(__A       ) as context:
																		self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'       )

											self.assertTrue(
											    context.exception.args[0].startswith(
											        'The `backend_tokenizer` provided does not match the expected format.'       )       )




				@require_ftfy
				def     a							(					self       ):
											super().test_tokenization_python_rust_equals()




				def     a							(					self       ):
											pass








from ..utils import DummyObject, requires_backends
class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Any     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Optional[int]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      str     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      List[Any]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Optional[int]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Dict     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Dict     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Union[str, Any]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Optional[int]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Optional[Any]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Any     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      Optional[Any]     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



class 	UpperCAmelCase_					(					metaclass=UpperCamelCase    ):

   '''simple docstring'''




   __A :      int     		=		["flax"]






   def __init__(		self						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(self						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )






   @classmethod
   def _snake_case    (		cls						,				*__A						,				**__A ):






     """simple docstring"""





     requires_backends(cls						,				["flax"] )



