
'''simple docstring'''




import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class  A				(							__snake_case    ,			unittest.TestCase						):
     __magic_name__						=		CLIPTokenizer
     __magic_name__						=		CLIPTokenizerFast
     __magic_name__						=		True
     __magic_name__						=		{}
     __magic_name__						=		False





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         super().setUp()

         # fmt: off
         A       :   List[Any]					=						['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
         # fmt: on
         A       :   Tuple					=						dict(zip(SCREAMING_SNAKE_CASE						,			range(len(SCREAMING_SNAKE_CASE	)	)	)	)
         A       :   Any					=						['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
         A       :   List[Any]					=						{'''unk_token''': '''<unk>'''}

         A       :   List[str]					=						os.path.join(self.tmpdirname						,			VOCAB_FILES_NAMES['''vocab_file''']	)
         A       :   int					=						os.path.join(self.tmpdirname						,			VOCAB_FILES_NAMES['''merges_file''']	)
         with open(self.vocab_file						,			'''w'''						,			encoding='''utf-8'''	) as fp:
             fp.write(json.dumps(SCREAMING_SNAKE_CASE	) + '''\n'''	)
         with open(self.merges_file						,			'''w'''						,			encoding='''utf-8'''	) as fp:
             fp.write('''\n'''.join(SCREAMING_SNAKE_CASE	)	)





     def 				__lowerCAmelCase							(							self						,			**SCREAMING_SNAKE_CASE	) ->				Union[str, Any]:






         """simple docstring"""






         kwargs.update(self.special_tokens_map	)
         return CLIPTokenizer.from_pretrained(self.tmpdirname						,			**SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self						,			**SCREAMING_SNAKE_CASE	) ->				Tuple:






         """simple docstring"""






         kwargs.update(self.special_tokens_map	)
         return CLIPTokenizerFast.from_pretrained(self.tmpdirname						,			**SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE	) ->				Union[str, Any]:






         """simple docstring"""






         A       :   str					=						'''lower newer'''
         A       :   int					=						'''lower newer'''
         return input_text, output_text





     def 				__lowerCAmelCase							(							self	) ->				Optional[int]:






         """simple docstring"""






         A       :   Tuple					=						CLIPTokenizer(self.vocab_file						,			self.merges_file						,			**self.special_tokens_map	)
         A       :   Optional[Any]					=						'''lower newer'''
         A       :   int					=						['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
         A       :   List[str]					=						tokenizer.tokenize(SCREAMING_SNAKE_CASE	)
         self.assertListEqual(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

         A       :   List[Any]					=						tokens + [tokenizer.unk_token]
         A       :   Any					=						[10, 2, 16, 9, 3, 2, 16, 20]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE	)						,			SCREAMING_SNAKE_CASE	)





     @require_ftfy
     def 				__lowerCAmelCase							(							self	) ->				List[Any]:






         """simple docstring"""






         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'	):
                 A       :   int					=						self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	)
                 A       :   List[Any]					=						self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	)

                 A       :   List[str]					=						'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
                 A       :   Any					=						tokenizer_s.tokenize(SCREAMING_SNAKE_CASE	)
                 A       :   List[Any]					=						tokenizer_r.tokenize(SCREAMING_SNAKE_CASE	)

                 self.assertListEqual(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

                 # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                 # with Tilde) encoded in 2 different ways
                 A       :   List[Any]					=						'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
                 A       :   Union[str, Any]					=						tokenizer_s.tokenize(SCREAMING_SNAKE_CASE	)
                 A       :   Dict					=						tokenizer_r.tokenize(SCREAMING_SNAKE_CASE	)

                 self.assertListEqual(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

                 # Test that the tokenization is identical on unicode of space type
                 A       :   Tuple					=						[
                     '''\u0009''',  # (horizontal tab, '\t')
                     '''\u000B''',  # (vertical tab)
                     '''\u000C''',  # (form feed)
                     '''\u0020''',  # (space, ' ')
                     '''\u200E''',  # (left-to-right mark):w
                     '''\u200F''',  # (right-to-left mark)
                 ]
                 for unicode_seq in spaces_unicodes:
                     A       :   List[str]					=						tokenizer_s.tokenize(SCREAMING_SNAKE_CASE	)
                     A       :   List[Any]					=						tokenizer_r.tokenize(SCREAMING_SNAKE_CASE	)

                     self.assertListEqual(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)

                 # Test that the tokenization is identical on unicode of line break type
                 A       :   int					=						[
                     '''\u000A''',  # (line feed, '\n')
                     '''\r\n''',  # (carriage return and line feed, '\r\n')
                     '''\u000D''',  # (carriage return, '\r')
                     '''\r''',  # (carriage return, '\r')
                     '''\u000D''',  # (carriage return, '\r')
                     '''\u2028''',  # (line separator)
                     '''\u2029''',  # (paragraph separator)
                     # "\u0085", # (next line)
                 ]

                 # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                 # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                 # space (and thus into an empty list).

                 for unicode_seq in line_break_unicodes:
                     A       :   List[str]					=						tokenizer_s.tokenize(SCREAMING_SNAKE_CASE	)
                     A       :   Tuple					=						tokenizer_r.tokenize(SCREAMING_SNAKE_CASE	)

                     self.assertListEqual(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self	) ->				Union[str, Any]:






         """simple docstring"""






         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'	):
                 A       :   int					=						'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
                 A       :   List[Any]					=						F'{text_of_1_token} {text_of_1_token}'

                 A       :   Optional[Any]					=						self.rust_tokenizer_class.from_pretrained(
                     SCREAMING_SNAKE_CASE						,			use_fast=SCREAMING_SNAKE_CASE						,			)
                 A       :   Optional[int]					=						tokenizer_r(SCREAMING_SNAKE_CASE						,			return_offsets_mapping=SCREAMING_SNAKE_CASE						,			add_special_tokens=SCREAMING_SNAKE_CASE	)
                 self.assertEqual(encoding.offset_mapping[0]						,			(0, len(SCREAMING_SNAKE_CASE	))	)
                 self.assertEqual(
                     encoding.offset_mapping[1]						,			(len(SCREAMING_SNAKE_CASE	) + 1, len(SCREAMING_SNAKE_CASE	) + 1 + len(SCREAMING_SNAKE_CASE	))						,			)

                 A       :   Any					=						F' {text}'

                 A       :   Dict					=						self.rust_tokenizer_class.from_pretrained(
                     SCREAMING_SNAKE_CASE						,			use_fast=SCREAMING_SNAKE_CASE						,			)
                 A       :   int					=						tokenizer_r(SCREAMING_SNAKE_CASE						,			return_offsets_mapping=SCREAMING_SNAKE_CASE						,			add_special_tokens=SCREAMING_SNAKE_CASE	)
                 self.assertEqual(encoding.offset_mapping[0]						,			(1, 1 + len(SCREAMING_SNAKE_CASE	))	)
                 self.assertEqual(
                     encoding.offset_mapping[1]						,			(1 + len(SCREAMING_SNAKE_CASE	) + 1, 1 + len(SCREAMING_SNAKE_CASE	) + 1 + len(SCREAMING_SNAKE_CASE	))						,			)





     def 				__lowerCAmelCase							(							self	) ->				Any:






         """simple docstring"""






         with self.assertRaises(SCREAMING_SNAKE_CASE	) as context:
             self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''	)

         self.assertTrue(
             context.exception.args[0].startswith(
                 '''The `backend_tokenizer` provided does not match the expected format.'''	)	)





     @require_ftfy
     def 				__lowerCAmelCase							(							self	) ->				Optional[Any]:






         """simple docstring"""






         super().test_tokenization_python_rust_equals()





     def 				__lowerCAmelCase							(							self	) ->				Optional[Any]:






         """simple docstring"""






         pass











"""simple docstring"""




from __future__ import annotations

import math

import numpy as np
from numpy.linalg import norm






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    np.ndarray    ,       _lowercase						:    np.ndarray					) ->float:






	'''simple docstring'''






	return math.sqrt(sum(pow(a - b    ,       2					) for a, b in zip(_lowercase    ,       _lowercase					)					)					)






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    np.ndarray    ,       _lowercase						:    np.ndarray					) ->list[list[list[float] | float]]:






	'''simple docstring'''






	if dataset.ndim != value_array.ndim:
		a      :		str	=				(
		    "Wrong input data's dimensions... "
		    F"""dataset : {dataset.ndim}, value_array : {value_array.ndim}"""
		)
		raise ValueError(_lowercase					)

	try:
		if dataset.shape[1] != value_array.shape[1]:
			a      :		int	=				(
			    "Wrong input data's shape... "
			    F"""dataset : {dataset.shape[1]}, value_array : {value_array.shape[1]}"""
			)
			raise ValueError(_lowercase					)
	except IndexError:
		if dataset.ndim != value_array.ndim:
			raise TypeError("Wrong shape"					)

	if dataset.dtype != value_array.dtype:
		a      :		Union[str, Any]	=				(
		    "Input data have different datatype... "
		    F"""dataset : {dataset.dtype}, value_array : {value_array.dtype}"""
		)
		raise TypeError(_lowercase					)

	a      :		str	=				[]

	for value in value_array:
		a      :		List[Any]	=				euclidean(_lowercase    ,       dataset[0]					)
		a      :		str	=				dataset[0].tolist()

		for dataset_value in dataset[1:]:
			a      :		Tuple	=				euclidean(_lowercase    ,       _lowercase					)

			if dist > temp_dist:
				a      :		Dict	=				temp_dist
				a      :		Optional[int]	=				dataset_value.tolist()

		answer.append([vector, dist]					)

	return answer






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    np.ndarray    ,       _lowercase						:    np.ndarray					) ->float:






	'''simple docstring'''






	return np.dot(_lowercase    ,       _lowercase					) / (norm(_lowercase					) * norm(_lowercase					))


if __name__ == "__main__":
		import doctest

		doctest.testmod()


