


import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
				from PIL import Image

				from transformers import CLIPImageProcessor, CLIPProcessor







@require_vision
class 	__lowercase    (unittest.TestCase  ):





							"""simple docstring"""







							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					tempfile.mkdtemp()

										# fmt: off
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
										# fmt: on
										SCREAMING_SNAKE_CASE_		:     str					=					dict(zip(lowerCAmelCase__			,						range(len(lowerCAmelCase__       )       )       )       )
										SCREAMING_SNAKE_CASE_		:     Any					=					['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>', '']
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					{'unk_token': '<unk>'}

										SCREAMING_SNAKE_CASE_		:     List[Any]					=					os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES['vocab_file']       )
										SCREAMING_SNAKE_CASE_		:     Dict					=					os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES['merges_file']       )
										with open(self.vocab_file			,						'w'			,						encoding='utf-8'       ) as fp:
													fp.write(json.dumps(lowerCAmelCase__       ) + '\n'       )
										with open(self.merges_file			,						'w'			,						encoding='utf-8'       ) as fp:
													fp.write('\n'.join(lowerCAmelCase__       )       )

										SCREAMING_SNAKE_CASE_		:     List[Any]					=					{
										    'do_resize': True,
										    'size': 2_0,
										    'do_center_crop': True,
										    'crop_size': 1_8,
										    'do_normalize': True,
										    'image_mean': [0.48_145_466, 0.4_578_275, 0.40_821_073],
										    'image_std': [0.26_862_954, 0.26_130_258, 0.27_577_711],
										}
										SCREAMING_SNAKE_CASE_		:     Dict					=					os.path.join(self.tmpdirname			,						lowerCAmelCase__       )
										with open(self.image_processor_file			,						'w'			,						encoding='utf-8'       ) as fp:
													json.dump(lowerCAmelCase__			,						lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self			,						**lowerCAmelCase__       ):





										"""simple docstring"""




										return CLIPTokenizer.from_pretrained(self.tmpdirname			,						**lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self			,						**lowerCAmelCase__       ):





										"""simple docstring"""




										return CLIPTokenizerFast.from_pretrained(self.tmpdirname			,						**lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self			,						**lowerCAmelCase__       ):





										"""simple docstring"""




										return CLIPImageProcessor.from_pretrained(self.tmpdirname			,						**lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										shutil.rmtree(self.tmpdirname       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					[np.random.randint(2_5_5			,						size=(3, 3_0, 4_0_0)			,						dtype=np.uinta       )]

										SCREAMING_SNAKE_CASE_		:     str					=					[Image.fromarray(np.moveaxis(lowerCAmelCase__			,						0			,						-1       )       ) for x in image_inputs]

										return image_inputs






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[str]					=					self.get_tokenizer()
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.get_rust_tokenizer()
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.get_image_processor()

										SCREAMING_SNAKE_CASE_		:     int					=					CLIPProcessor(tokenizer=lowerCAmelCase__			,						image_processor=lowerCAmelCase__       )
										processor_slow.save_pretrained(self.tmpdirname       )
										SCREAMING_SNAKE_CASE_		:     Any					=					CLIPProcessor.from_pretrained(self.tmpdirname			,						use_fast=lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     Dict					=					CLIPProcessor(tokenizer=lowerCAmelCase__			,						image_processor=lowerCAmelCase__       )
										processor_fast.save_pretrained(self.tmpdirname       )
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					CLIPProcessor.from_pretrained(self.tmpdirname       )

										self.assertEqual(processor_slow.tokenizer.get_vocab()			,						tokenizer_slow.get_vocab()       )
										self.assertEqual(processor_fast.tokenizer.get_vocab()			,						tokenizer_fast.get_vocab()       )
										self.assertEqual(tokenizer_slow.get_vocab()			,						tokenizer_fast.get_vocab()       )
										self.assertIsInstance(processor_slow.tokenizer			,						lowerCAmelCase__       )
										self.assertIsInstance(processor_fast.tokenizer			,						lowerCAmelCase__       )

										self.assertEqual(processor_slow.image_processor.to_json_string()			,						image_processor.to_json_string()       )
										self.assertEqual(processor_fast.image_processor.to_json_string()			,						image_processor.to_json_string()       )
										self.assertIsInstance(processor_slow.image_processor			,						lowerCAmelCase__       )
										self.assertIsInstance(processor_fast.image_processor			,						lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     int					=					CLIPProcessor(tokenizer=self.get_tokenizer()			,						image_processor=self.get_image_processor()       )
										processor.save_pretrained(self.tmpdirname       )

										SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.get_tokenizer(bos_token='(BOS)'			,						eos_token='(EOS)'       )
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.get_image_processor(do_normalize=lowerCAmelCase__			,						padding_value=1.0       )

										SCREAMING_SNAKE_CASE_		:     List[str]					=					CLIPProcessor.from_pretrained(
										    self.tmpdirname			,						bos_token='(BOS)'			,						eos_token='(EOS)'			,						do_normalize=lowerCAmelCase__			,						padding_value=1.0       )

										self.assertEqual(processor.tokenizer.get_vocab()			,						tokenizer_add_kwargs.get_vocab()       )
										self.assertIsInstance(processor.tokenizer			,						lowerCAmelCase__       )

										self.assertEqual(processor.image_processor.to_json_string()			,						image_processor_add_kwargs.to_json_string()       )
										self.assertIsInstance(processor.image_processor			,						lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.get_image_processor()
										SCREAMING_SNAKE_CASE_		:     str					=					self.get_tokenizer()

										SCREAMING_SNAKE_CASE_		:     Any					=					CLIPProcessor(tokenizer=lowerCAmelCase__			,						image_processor=lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     List[str]					=					self.prepare_image_inputs()

										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					image_processor(lowerCAmelCase__			,						return_tensors='np'       )
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					processor(images=lowerCAmelCase__			,						return_tensors='np'       )

										for key in input_image_proc.keys():
													self.assertAlmostEqual(input_image_proc[key].sum()			,						input_processor[key].sum()			,						delta=1E-2       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     str					=					self.get_image_processor()
										SCREAMING_SNAKE_CASE_		:     str					=					self.get_tokenizer()

										SCREAMING_SNAKE_CASE_		:     str					=					CLIPProcessor(tokenizer=lowerCAmelCase__			,						image_processor=lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     List[Any]					=					'lower newer'

										SCREAMING_SNAKE_CASE_		:     Any					=					processor(text=lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     List[str]					=					tokenizer(lowerCAmelCase__       )

										for key in encoded_tok.keys():
													self.assertListEqual(encoded_tok[key]			,						encoded_processor[key]       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					self.get_image_processor()
										SCREAMING_SNAKE_CASE_		:     Any					=					self.get_tokenizer()

										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					CLIPProcessor(tokenizer=lowerCAmelCase__			,						image_processor=lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     List[str]					=					'lower newer'
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					self.prepare_image_inputs()

										SCREAMING_SNAKE_CASE_		:     Any					=					processor(text=lowerCAmelCase__			,						images=lowerCAmelCase__       )

										self.assertListEqual(list(inputs.keys()       )			,						['input_ids', 'attention_mask', 'pixel_values']       )

										# test if it raises when no input is passed
										with pytest.raises(lowerCAmelCase__       ):
													processor()






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.get_image_processor()
										SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.get_tokenizer()

										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					CLIPProcessor(tokenizer=lowerCAmelCase__			,						image_processor=lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     List[Any]					=					[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

										SCREAMING_SNAKE_CASE_		:     List[Any]					=					processor.batch_decode(lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					tokenizer.batch_decode(lowerCAmelCase__       )

										self.assertListEqual(lowerCAmelCase__			,						lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Dict					=					self.get_image_processor()
										SCREAMING_SNAKE_CASE_		:     Tuple					=					self.get_tokenizer()

										SCREAMING_SNAKE_CASE_		:     str					=					CLIPProcessor(tokenizer=lowerCAmelCase__			,						image_processor=lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     int					=					'lower newer'
										SCREAMING_SNAKE_CASE_		:     str					=					self.prepare_image_inputs()

										SCREAMING_SNAKE_CASE_		:     Tuple					=					processor(text=lowerCAmelCase__			,						images=lowerCAmelCase__       )

										self.assertListEqual(list(inputs.keys()       )			,						processor.model_input_names       )










import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class    UpperCAmelCase__(				lowerCamelCase  ,				unittest.TestCase						):

   '''simple docstring'''




   A     :	int												=     CTRLTokenizer
   A     :	Tuple												=     False
   A     :	Any												=     False


   def       UpperCAmelCase		(    self	:      int)						->					Any:




         """simple docstring"""

         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         lowercase__  =	['adapt', 're@@', 'a@@', 'apt', 'c@@', 't', '<unk>']
         lowercase__  =	dict(zip(lowerCAmelCase      ,					range(len(lowerCAmelCase))))
         lowercase__  =	['#version: 0.2', 'a p', 'ap t</w>', 'r e', 'a d', 'ad apt</w>', '']
         lowercase__  =	{'unk_token': '<unk>'}

         lowercase__  =	os.path.join(self.tmpdirname      ,					VOCAB_FILES_NAMES['vocab_file'])
         lowercase__  =	os.path.join(self.tmpdirname      ,					VOCAB_FILES_NAMES['merges_file'])
         with open(self.vocab_file      ,					'w'      ,					encoding='utf-8') as fp:
               fp.write(json.dumps(lowerCAmelCase) + '\n')
         with open(self.merges_file      ,					'w'      ,					encoding='utf-8') as fp:
               fp.write('\n'.join(lowerCAmelCase))


   def       UpperCAmelCase		(    self	:      int      ,					**lowerCAmelCase	:      str)						->					Dict:




         """simple docstring"""

         kwargs.update(self.special_tokens_map)
         return CTRLTokenizer.from_pretrained(self.tmpdirname      ,					**lowerCAmelCase)


   def       UpperCAmelCase		(    self	:      List[str]      ,					lowerCAmelCase	:      Optional[int])						->					Optional[Any]:




         """simple docstring"""

         lowercase__  =	'adapt react readapt apt'
         lowercase__  =	'adapt react readapt apt'
         return input_text, output_text



   def       UpperCAmelCase		(    self	:      Optional[int])						->					str:




         """simple docstring"""

         lowercase__  =	CTRLTokenizer(self.vocab_file      ,					self.merges_file      ,					**self.special_tokens_map)
         lowercase__  =	'adapt react readapt apt'
         lowercase__  =	'adapt re@@ a@@ c@@ t re@@ adapt apt'.split()
         lowercase__  =	tokenizer.tokenize(lowerCAmelCase)
         self.assertListEqual(lowerCAmelCase      ,					lowerCAmelCase)

         lowercase__  =	tokens + [tokenizer.unk_token]

         lowercase__  =	[0, 1, 2, 4, 5, 1, 0, 3, 6]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)      ,					lowerCAmelCase)
