


'''simple docstring'''







import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 					lowercase_     (    a__  ,						unittest.TestCase   ):
    __UpperCAmelCase   			=	CLIPTokenizer
    __UpperCAmelCase   			=	CLIPTokenizerFast
    __UpperCAmelCase   			=	True
    __UpperCAmelCase   			=	{}
    __UpperCAmelCase   			=	False


    def    __a							(     self    ):
          super().setUp()

          # fmt: off
          UpperCamelCase__										=				["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
          # fmt: on
          UpperCamelCase__										=				dict(zip(a  ,					range(len(a    )    )    )    )
          UpperCamelCase__										=				["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
          UpperCamelCase__										=				{"unk_token": "<unk>"}

          UpperCamelCase__										=				os.path.join(self.tmpdirname  ,					VOCAB_FILES_NAMES["vocab_file"]    )
          UpperCamelCase__										=				os.path.join(self.tmpdirname  ,					VOCAB_FILES_NAMES["merges_file"]    )
          with open(self.vocab_file  ,					"w"  ,					encoding="utf-8"    ) as fp:
                fp.write(json.dumps(a    ) + "\n"    )
          with open(self.merges_file  ,					"w"  ,					encoding="utf-8"    ) as fp:
                fp.write("\n".join(a    )    )


    def    __a							(     self  ,					**a    ):
          kwargs.update(self.special_tokens_map    )
          return CLIPTokenizer.from_pretrained(self.tmpdirname  ,					**a    )


    def    __a							(     self  ,					**a    ):
          kwargs.update(self.special_tokens_map    )
          return CLIPTokenizerFast.from_pretrained(self.tmpdirname  ,					**a    )


    def    __a							(     self  ,					a    ):
          UpperCamelCase__										=				"lower newer"
          UpperCamelCase__										=				"lower newer"
          return input_text, output_text


    def    __a							(     self    ):
          UpperCamelCase__										=				CLIPTokenizer(self.vocab_file  ,					self.merges_file  ,					**self.special_tokens_map    )
          UpperCamelCase__										=				"lower newer"
          UpperCamelCase__										=				["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
          UpperCamelCase__										=				tokenizer.tokenize(a    )
          self.assertListEqual(a  ,					a    )

          UpperCamelCase__										=				tokens + [tokenizer.unk_token]
          UpperCamelCase__										=				[10, 2, 16, 9, 3, 2, 16, 20]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(a    )  ,					a    )


    @require_ftfy
    def    __a							(     self    ):
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
                      UpperCamelCase__										=				self.tokenizer_class.from_pretrained(a  ,					**a    )
                      UpperCamelCase__										=				self.rust_tokenizer_class.from_pretrained(a  ,					**a    )

                      UpperCamelCase__										=				"A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
                      UpperCamelCase__										=				tokenizer_s.tokenize(a    )
                      UpperCamelCase__										=				tokenizer_r.tokenize(a    )

                      self.assertListEqual(a  ,					a    )

                      # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                      # with Tilde) encoded in 2 different ways
                      UpperCamelCase__										=				"xa\u0303y" + " " + "x\xe3y"
                      UpperCamelCase__										=				tokenizer_s.tokenize(a    )
                      UpperCamelCase__										=				tokenizer_r.tokenize(a    )

                      self.assertListEqual(a  ,					a    )

                      # Test that the tokenization is identical on unicode of space type
                      UpperCamelCase__										=				[
                          "\u0009",  # (horizontal tab, '\t')
                          "\u000B",  # (vertical tab)
                          "\u000C",  # (form feed)
                          "\u0020",  # (space, ' ')
                          "\u200E",  # (left-to-right mark):w
                          "\u200F",  # (right-to-left mark)
                      ]
                      for unicode_seq in spaces_unicodes:
                            UpperCamelCase__										=				tokenizer_s.tokenize(a    )
                            UpperCamelCase__										=				tokenizer_r.tokenize(a    )

                            self.assertListEqual(a  ,					a    )

                      # Test that the tokenization is identical on unicode of line break type
                      UpperCamelCase__										=				[
                          "\u000A",  # (line feed, '\n')
                          "\r\n",  # (carriage return and line feed, '\r\n')
                          "\u000D",  # (carriage return, '\r')
                          "\r",  # (carriage return, '\r')
                          "\u000D",  # (carriage return, '\r')
                          "\u2028",  # (line separator)
                          "\u2029",  # (paragraph separator)
                          # "\u0085", # (next line)
                      ]

                      # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                      # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                      # space (and thus into an empty list).

                      for unicode_seq in line_break_unicodes:
                            UpperCamelCase__										=				tokenizer_s.tokenize(a    )
                            UpperCamelCase__										=				tokenizer_r.tokenize(a    )

                            self.assertListEqual(a  ,					a    )


    def    __a							(     self    ):
          # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
                      UpperCamelCase__										=				"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
                      UpperCamelCase__										=				f'''{text_of_1_token} {text_of_1_token}'''

                      UpperCamelCase__										=				self.rust_tokenizer_class.from_pretrained(
                          a  ,					use_fast=a  ,					)
                      UpperCamelCase__										=				tokenizer_r(a  ,					return_offsets_mapping=a  ,					add_special_tokens=a    )
                      self.assertEqual(encoding.offset_mapping[0]  ,					(0, len(a    ))    )
                      self.assertEqual(
                          encoding.offset_mapping[1]  ,					(len(a    ) + 1, len(a    ) + 1 + len(a    ))  ,					)

                      UpperCamelCase__										=				f''' {text}'''

                      UpperCamelCase__										=				self.rust_tokenizer_class.from_pretrained(
                          a  ,					use_fast=a  ,					)
                      UpperCamelCase__										=				tokenizer_r(a  ,					return_offsets_mapping=a  ,					add_special_tokens=a    )
                      self.assertEqual(encoding.offset_mapping[0]  ,					(1, 1 + len(a    ))    )
                      self.assertEqual(
                          encoding.offset_mapping[1]  ,					(1 + len(a    ) + 1, 1 + len(a    ) + 1 + len(a    ))  ,					)


    def    __a							(     self    ):
          # Test related to the breaking change introduced in transformers v4.17.0
          # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
          with self.assertRaises(a    ) as context:
                self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"    )

          self.assertTrue(
              context.exception.args[0].startswith(
                  "The `backend_tokenizer` provided does not match the expected format."    )    )


    @require_ftfy
    def    __a							(     self    ):
          super().test_tokenization_python_rust_equals()


    def    __a							(     self    ):
          # CLIP always lower cases letters
          pass










'''simple docstring'''




# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce



def       __lowerCamelCase  (					_lowercase		)      ->			Tuple:
							return (torch.arange(state.num_processes		) + 1.0 + (state.num_processes * state.process_index)).to(state.device		)



def       __lowerCamelCase  (					_lowercase		)      ->			List[Any]:
							UpperCAmelCase     :		Any							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		Union[str, Any]							     =		gather(_lowercase		)
							assert gathered_tensor.tolist() == list(range(1	,				state.num_processes**2 + 1		)		)



def       __lowerCamelCase  (					_lowercase		)      ->			Optional[int]:
							UpperCAmelCase     :		Any							     =		[state.process_index]
							UpperCAmelCase     :		Union[str, Any]							     =		gather_object(_lowercase		)
							assert len(_lowercase		) == state.num_processes, F'''{gathered_obj}, {len(_lowercase		)} != {state.num_processes}'''
							assert gathered_obj == list(range(state.num_processes		)		), F'''{gathered_obj} != {list(range(state.num_processes		)		)}'''



def       __lowerCamelCase  (					_lowercase		)      ->			List[Any]:
							UpperCAmelCase     :		Optional[int]							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		List[str]							     =		broadcast(_lowercase		)
							assert broadcasted_tensor.shape == torch.Size([state.num_processes]		)
							assert broadcasted_tensor.tolist() == list(range(1	,				state.num_processes + 1		)		)



def       __lowerCamelCase  (					_lowercase		)      ->			Tuple:
							# We need to pad the tensor with one more element if we are the main process
							# to ensure that we can pad
							if state.is_main_process:
														UpperCAmelCase     :		Optional[Any]							     =		torch.arange(state.num_processes + 1		).to(state.device		)
							else:
														UpperCAmelCase     :		Tuple							     =		torch.arange(state.num_processes		).to(state.device		)
							UpperCAmelCase     :		Optional[Any]							     =		pad_across_processes(_lowercase		)
							assert padded_tensor.shape == torch.Size([state.num_processes + 1]		)
							if not state.is_main_process:
														assert padded_tensor.tolist() == list(range(0	,				state.num_processes		)		) + [0]



def       __lowerCamelCase  (					_lowercase		)      ->			Dict:
							# For now runs on only two processes
							if state.num_processes != 2:
														return
							UpperCAmelCase     :		Optional[Any]							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		Optional[Any]							     =		reduce(_lowercase	,				"""sum"""		)
							UpperCAmelCase     :		Optional[Any]							     =		torch.tensor([4.0, 6]		).to(state.device		)
							assert torch.allclose(_lowercase	,				_lowercase		), F'''{reduced_tensor} != {truth_tensor}'''



def       __lowerCamelCase  (					_lowercase		)      ->			Optional[Any]:
							# For now runs on only two processes
							if state.num_processes != 2:
														return
							UpperCAmelCase     :		Tuple							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		Optional[int]							     =		reduce(_lowercase	,				"""mean"""		)
							UpperCAmelCase     :		str							     =		torch.tensor([2.0, 3]		).to(state.device		)
							assert torch.allclose(_lowercase	,				_lowercase		), F'''{reduced_tensor} != {truth_tensor}'''



def       __lowerCamelCase  (					_lowercase		)      ->			Optional[int]:
							# For xla_spawn (TPUs)
							main()



def       __lowerCamelCase  (					)      ->			int:
							UpperCAmelCase     :		List[Any]							     =		PartialState()
							state.print(F'''State: {state}'''		)
							state.print("""testing gather"""		)
							test_gather(_lowercase		)
							state.print("""testing gather_object"""		)
							test_gather_object(_lowercase		)
							state.print("""testing broadcast"""		)
							test_broadcast(_lowercase		)
							state.print("""testing pad_across_processes"""		)
							test_pad_across_processes(_lowercase		)
							state.print("""testing reduce_sum"""		)
							test_reduce_sum(_lowercase		)
							state.print("""testing reduce_mean"""		)
							test_reduce_mean(_lowercase		)


if __name__ == "__main__":
							main()

