






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  UpperCamelCase_      (			snake_case_       ,					unittest.TestCase				):



	'''simple docstring'''




	lowerCAmelCase													= CLIPTokenizer
	lowerCAmelCase													= CLIPTokenizerFast
	lowerCAmelCase													= True
	lowerCAmelCase													= {}
	lowerCAmelCase													= False




	def 							_UpperCamelCase			(       self					)       ->    Optional[int]:
								super().setUp()

								# fmt: off
								snake_case_				=							["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
								# fmt: on
								snake_case_				=							dict(zip(a       ,			range(len(a					)					)					)					)
								snake_case_				=							["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
								snake_case_				=							{"""unk_token""": """<unk>"""}

								snake_case_				=							os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES['vocab_file']					)
								snake_case_				=							os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES['merges_file']					)
								with open(self.vocab_file       ,			'w'       ,			encoding='utf-8'					) as fp:
															fp.write(json.dumps(a					) + '\n'					)
								with open(self.merges_file       ,			'w'       ,			encoding='utf-8'					) as fp:
															fp.write('\n'.join(a					)					)




	def 							_UpperCamelCase			(       self       ,			**a					)       ->    str:
								kwargs.update(self.special_tokens_map					)
								return CLIPTokenizer.from_pretrained(self.tmpdirname       ,			**a					)




	def 							_UpperCamelCase			(       self       ,			**a					)       ->    Union[str, Any]:
								kwargs.update(self.special_tokens_map					)
								return CLIPTokenizerFast.from_pretrained(self.tmpdirname       ,			**a					)




	def 							_UpperCamelCase			(       self       ,			a					)       ->    Optional[int]:
								snake_case_				=							"""lower newer"""
								snake_case_				=							"""lower newer"""
								return input_text, output_text




	def 							_UpperCamelCase			(       self					)       ->    Any:
								snake_case_				=							CLIPTokenizer(self.vocab_file       ,			self.merges_file       ,			**self.special_tokens_map					)
								snake_case_				=							"""lower newer"""
								snake_case_				=							["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
								snake_case_				=							tokenizer.tokenize(a					)
								self.assertListEqual(a       ,			a					)

								snake_case_				=							tokens + [tokenizer.unk_token]
								snake_case_				=							[10, 2, 16, 9, 3, 2, 16, 20]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a					)       ,			a					)




	@require_ftfy
	def 							_UpperCamelCase			(       self					)       ->    Dict:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																						snake_case_				=							self.tokenizer_class.from_pretrained(a       ,			**a					)
																						snake_case_				=							self.rust_tokenizer_class.from_pretrained(a       ,			**a					)

																						snake_case_				=							"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
																						snake_case_				=							tokenizer_s.tokenize(a					)
																						snake_case_				=							tokenizer_r.tokenize(a					)

																						self.assertListEqual(a       ,			a					)

																						# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																						# with Tilde) encoded in 2 different ways
																						snake_case_				=							"""xa\u0303y""" + """ """ + """x\xe3y"""
																						snake_case_				=							tokenizer_s.tokenize(a					)
																						snake_case_				=							tokenizer_r.tokenize(a					)

																						self.assertListEqual(a       ,			a					)

																						# Test that the tokenization is identical on unicode of space type
																						snake_case_				=							[
																						    """\u0009""",  # (horizontal tab, '\t')
																						    """\u000B""",  # (vertical tab)
																						    """\u000C""",  # (form feed)
																						    """\u0020""",  # (space, ' ')
																						    """\u200E""",  # (left-to-right mark):w
																						    """\u200F""",  # (right-to-left mark)
																						]
																						for unicode_seq in spaces_unicodes:
																													snake_case_				=							tokenizer_s.tokenize(a					)
																													snake_case_				=							tokenizer_r.tokenize(a					)

																													self.assertListEqual(a       ,			a					)

																						# Test that the tokenization is identical on unicode of line break type
																						snake_case_				=							[
																						    """\u000A""",  # (line feed, '\n')
																						    """\r\n""",  # (carriage return and line feed, '\r\n')
																						    """\u000D""",  # (carriage return, '\r')
																						    """\r""",  # (carriage return, '\r')
																						    """\u000D""",  # (carriage return, '\r')
																						    """\u2028""",  # (line separator)
																						    """\u2029""",  # (paragraph separator)
																						    # "\u0085", # (next line)
																						]

																						# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																						# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																						# space (and thus into an empty list).

																						for unicode_seq in line_break_unicodes:
																													snake_case_				=							tokenizer_s.tokenize(a					)
																													snake_case_				=							tokenizer_r.tokenize(a					)

																													self.assertListEqual(a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Union[str, Any]:
								# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																						snake_case_				=							"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
																						snake_case_				=							F'''{text_of_1_token} {text_of_1_token}'''

																						snake_case_				=							self.rust_tokenizer_class.from_pretrained(
																						    a       ,			use_fast=a       ,			)
																						snake_case_				=							tokenizer_r(a       ,			return_offsets_mapping=a       ,			add_special_tokens=a					)
																						self.assertEqual(encoding.offset_mapping[0]       ,			(0, len(a					))					)
																						self.assertEqual(
																						    encoding.offset_mapping[1]       ,			(len(a					) + 1, len(a					) + 1 + len(a					))       ,			)

																						snake_case_				=							F''' {text}'''

																						snake_case_				=							self.rust_tokenizer_class.from_pretrained(
																						    a       ,			use_fast=a       ,			)
																						snake_case_				=							tokenizer_r(a       ,			return_offsets_mapping=a       ,			add_special_tokens=a					)
																						self.assertEqual(encoding.offset_mapping[0]       ,			(1, 1 + len(a					))					)
																						self.assertEqual(
																						    encoding.offset_mapping[1]       ,			(1 + len(a					) + 1, 1 + len(a					) + 1 + len(a					))       ,			)




	def 							_UpperCamelCase			(       self					)       ->    Optional[Any]:
								# Test related to the breaking change introduced in transformers v4.17.0
								# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
								with self.assertRaises(a					) as context:
															self.rust_tokenizer_class.from_pretrained('robot-test/old-clip-tokenizer'					)

								self.assertTrue(
								    context.exception.args[0].startswith(
								        'The `backend_tokenizer` provided does not match the expected format.'					)					)




	@require_ftfy
	def 							_UpperCamelCase			(       self					)       ->    Any:
								super().test_tokenization_python_rust_equals()






	def 							_UpperCamelCase			(       self					)       ->    int:
								# CLIP always lower cases letters
								pass










'''simple docstring'''




# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce



def       __lowerCamelCase  (					_lowercase		)      ->			Tuple:
							return (torch.arange(state.num_processes		) + 1.0 + (state.num_processes * state.process_index)).to(state.device		)



def       __lowerCamelCase  (					_lowercase		)      ->			List[Any]:
							UpperCAmelCase     :		Any							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		Union[str, Any]							     =		gather(_lowercase		)
							assert gathered_tensor.tolist() == list(range(1	,				state.num_processes**2 + 1		)		)



def       __lowerCamelCase  (					_lowercase		)      ->			Optional[int]:
							UpperCAmelCase     :		Any							     =		[state.process_index]
							UpperCAmelCase     :		Union[str, Any]							     =		gather_object(_lowercase		)
							assert len(_lowercase		) == state.num_processes, F'''{gathered_obj}, {len(_lowercase		)} != {state.num_processes}'''
							assert gathered_obj == list(range(state.num_processes		)		), F'''{gathered_obj} != {list(range(state.num_processes		)		)}'''



def       __lowerCamelCase  (					_lowercase		)      ->			List[Any]:
							UpperCAmelCase     :		Optional[int]							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		List[str]							     =		broadcast(_lowercase		)
							assert broadcasted_tensor.shape == torch.Size([state.num_processes]		)
							assert broadcasted_tensor.tolist() == list(range(1	,				state.num_processes + 1		)		)



def       __lowerCamelCase  (					_lowercase		)      ->			Tuple:
							# We need to pad the tensor with one more element if we are the main process
							# to ensure that we can pad
							if state.is_main_process:
														UpperCAmelCase     :		Optional[Any]							     =		torch.arange(state.num_processes + 1		).to(state.device		)
							else:
														UpperCAmelCase     :		Tuple							     =		torch.arange(state.num_processes		).to(state.device		)
							UpperCAmelCase     :		Optional[Any]							     =		pad_across_processes(_lowercase		)
							assert padded_tensor.shape == torch.Size([state.num_processes + 1]		)
							if not state.is_main_process:
														assert padded_tensor.tolist() == list(range(0	,				state.num_processes		)		) + [0]



def       __lowerCamelCase  (					_lowercase		)      ->			Dict:
							# For now runs on only two processes
							if state.num_processes != 2:
														return
							UpperCAmelCase     :		Optional[Any]							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		Optional[Any]							     =		reduce(_lowercase	,				"""sum"""		)
							UpperCAmelCase     :		Optional[Any]							     =		torch.tensor([4.0, 6]		).to(state.device		)
							assert torch.allclose(_lowercase	,				_lowercase		), F'''{reduced_tensor} != {truth_tensor}'''



def       __lowerCamelCase  (					_lowercase		)      ->			Optional[Any]:
							# For now runs on only two processes
							if state.num_processes != 2:
														return
							UpperCAmelCase     :		Tuple							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		Optional[int]							     =		reduce(_lowercase	,				"""mean"""		)
							UpperCAmelCase     :		str							     =		torch.tensor([2.0, 3]		).to(state.device		)
							assert torch.allclose(_lowercase	,				_lowercase		), F'''{reduced_tensor} != {truth_tensor}'''



def       __lowerCamelCase  (					_lowercase		)      ->			Optional[int]:
							# For xla_spawn (TPUs)
							main()



def       __lowerCamelCase  (					)      ->			int:
							UpperCAmelCase     :		List[Any]							     =		PartialState()
							state.print(F'''State: {state}'''		)
							state.print("""testing gather"""		)
							test_gather(_lowercase		)
							state.print("""testing gather_object"""		)
							test_gather_object(_lowercase		)
							state.print("""testing broadcast"""		)
							test_broadcast(_lowercase		)
							state.print("""testing pad_across_processes"""		)
							test_pad_across_processes(_lowercase		)
							state.print("""testing reduce_sum"""		)
							test_reduce_sum(_lowercase		)
							state.print("""testing reduce_mean"""		)
							test_reduce_mean(_lowercase		)


if __name__ == "__main__":
							main()

