


import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
						from PIL import Image

						from transformers import CLIPImageProcessor, CLIPProcessor
@require_vision
class        _lowercase      (			unittest.TestCase       ):






				'''simple docstring'''



				def 		__magic_name__(       self			:List[Any]	)		->			Optional[int]:
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		tempfile.mkdtemp()

										# fmt: off
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
										# fmt: on
										__SCREAMING_SNAKE_CASE						:     str					=		dict(zip(lowerCAmelCase__			,			range(len(lowerCAmelCase__	)	)	)	)
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''', '''''']
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		{'''unk_token''': '''<unk>'''}

										__SCREAMING_SNAKE_CASE						:     str					=		os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES['''vocab_file''']	)
										__SCREAMING_SNAKE_CASE						:     Dict					=		os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES['''merges_file''']	)
										with open(self.vocab_file			,			'''w'''			,			encoding='''utf-8'''	) as fp:
																fp.write(json.dumps(lowerCAmelCase__	) + '''\n'''	)
										with open(self.merges_file			,			'''w'''			,			encoding='''utf-8'''	) as fp:
																fp.write('''\n'''.join(lowerCAmelCase__	)	)

										__SCREAMING_SNAKE_CASE						:     Tuple					=		{
										    '''do_resize''': True,
										    '''size''': 20,
										    '''do_center_crop''': True,
										    '''crop_size''': 18,
										    '''do_normalize''': True,
										    '''image_mean''': [0.4814_5466, 0.457_8275, 0.4082_1073],
										    '''image_std''': [0.2686_2954, 0.2613_0258, 0.2757_7711],
										}
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		os.path.join(self.tmpdirname			,			lowerCAmelCase__	)
										with open(self.image_processor_file			,			'''w'''			,			encoding='''utf-8'''	) as fp:
																json.dump(lowerCAmelCase__			,			lowerCAmelCase__	)


				def 		__magic_name__(       self			:Optional[int]			,			**lowerCAmelCase__			:Union[str, Any]	)		->			List[str]:
										return CLIPTokenizer.from_pretrained(self.tmpdirname			,			**lowerCAmelCase__	)


				def 		__magic_name__(       self			:int			,			**lowerCAmelCase__			:Union[str, Any]	)		->			Any:
										return CLIPTokenizerFast.from_pretrained(self.tmpdirname			,			**lowerCAmelCase__	)


				def 		__magic_name__(       self			:List[Any]			,			**lowerCAmelCase__			:int	)		->			Union[str, Any]:
										return CLIPImageProcessor.from_pretrained(self.tmpdirname			,			**lowerCAmelCase__	)


				def 		__magic_name__(       self			:Optional[Any]	)		->			List[str]:
										shutil.rmtree(self.tmpdirname	)


				def 		__magic_name__(       self			:Tuple	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     Tuple					=		[np.random.randint(255			,			size=(3, 30, 400)			,			dtype=np.uinta	)]

										__SCREAMING_SNAKE_CASE						:     int					=		[Image.fromarray(np.moveaxis(lowerCAmelCase__			,			0			,			-1	)	) for x in image_inputs]

										return image_inputs


				def 		__magic_name__(       self			:List[Any]	)		->			Tuple:
										__SCREAMING_SNAKE_CASE						:     List[str]					=		self.get_tokenizer()
										__SCREAMING_SNAKE_CASE						:     str					=		self.get_rust_tokenizer()
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		self.get_image_processor()

										__SCREAMING_SNAKE_CASE						:     str					=		CLIPProcessor(tokenizer=lowerCAmelCase__			,			image_processor=lowerCAmelCase__	)
										processor_slow.save_pretrained(self.tmpdirname	)
										__SCREAMING_SNAKE_CASE						:     str					=		CLIPProcessor.from_pretrained(self.tmpdirname			,			use_fast=lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     int					=		CLIPProcessor(tokenizer=lowerCAmelCase__			,			image_processor=lowerCAmelCase__	)
										processor_fast.save_pretrained(self.tmpdirname	)
										__SCREAMING_SNAKE_CASE						:     Tuple					=		CLIPProcessor.from_pretrained(self.tmpdirname	)

										self.assertEqual(processor_slow.tokenizer.get_vocab()			,			tokenizer_slow.get_vocab()	)
										self.assertEqual(processor_fast.tokenizer.get_vocab()			,			tokenizer_fast.get_vocab()	)
										self.assertEqual(tokenizer_slow.get_vocab()			,			tokenizer_fast.get_vocab()	)
										self.assertIsInstance(processor_slow.tokenizer			,			lowerCAmelCase__	)
										self.assertIsInstance(processor_fast.tokenizer			,			lowerCAmelCase__	)

										self.assertEqual(processor_slow.image_processor.to_json_string()			,			image_processor.to_json_string()	)
										self.assertEqual(processor_fast.image_processor.to_json_string()			,			image_processor.to_json_string()	)
										self.assertIsInstance(processor_slow.image_processor			,			lowerCAmelCase__	)
										self.assertIsInstance(processor_fast.image_processor			,			lowerCAmelCase__	)


				def 		__magic_name__(       self			:Optional[int]	)		->			Union[str, Any]:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		CLIPProcessor(tokenizer=self.get_tokenizer()			,			image_processor=self.get_image_processor()	)
										processor.save_pretrained(self.tmpdirname	)

										__SCREAMING_SNAKE_CASE						:     Tuple					=		self.get_tokenizer(bos_token='''(BOS)'''			,			eos_token='''(EOS)'''	)
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		self.get_image_processor(do_normalize=lowerCAmelCase__			,			padding_value=1.0	)

										__SCREAMING_SNAKE_CASE						:     int					=		CLIPProcessor.from_pretrained(
										    self.tmpdirname			,			bos_token='''(BOS)'''			,			eos_token='''(EOS)'''			,			do_normalize=lowerCAmelCase__			,			padding_value=1.0	)

										self.assertEqual(processor.tokenizer.get_vocab()			,			tokenizer_add_kwargs.get_vocab()	)
										self.assertIsInstance(processor.tokenizer			,			lowerCAmelCase__	)

										self.assertEqual(processor.image_processor.to_json_string()			,			image_processor_add_kwargs.to_json_string()	)
										self.assertIsInstance(processor.image_processor			,			lowerCAmelCase__	)


				def 		__magic_name__(       self			:Optional[int]	)		->			Optional[int]:
										__SCREAMING_SNAKE_CASE						:     Dict					=		self.get_image_processor()
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		self.get_tokenizer()

										__SCREAMING_SNAKE_CASE						:     Dict					=		CLIPProcessor(tokenizer=lowerCAmelCase__			,			image_processor=lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     Dict					=		self.prepare_image_inputs()

										__SCREAMING_SNAKE_CASE						:     int					=		image_processor(lowerCAmelCase__			,			return_tensors='''np'''	)
										__SCREAMING_SNAKE_CASE						:     Any					=		processor(images=lowerCAmelCase__			,			return_tensors='''np'''	)

										for key in input_image_proc.keys():
																self.assertAlmostEqual(input_image_proc[key].sum()			,			input_processor[key].sum()			,			delta=1E-2	)


				def 		__magic_name__(       self			:Optional[int]	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     Any					=		self.get_image_processor()
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		self.get_tokenizer()

										__SCREAMING_SNAKE_CASE						:     List[Any]					=		CLIPProcessor(tokenizer=lowerCAmelCase__			,			image_processor=lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     str					=		'''lower newer'''

										__SCREAMING_SNAKE_CASE						:     List[str]					=		processor(text=lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     List[str]					=		tokenizer(lowerCAmelCase__	)

										for key in encoded_tok.keys():
																self.assertListEqual(encoded_tok[key]			,			encoded_processor[key]	)


				def 		__magic_name__(       self			:Optional[Any]	)		->			List[str]:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		self.get_image_processor()
										__SCREAMING_SNAKE_CASE						:     Tuple					=		self.get_tokenizer()

										__SCREAMING_SNAKE_CASE						:     List[Any]					=		CLIPProcessor(tokenizer=lowerCAmelCase__			,			image_processor=lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		'''lower newer'''
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		self.prepare_image_inputs()

										__SCREAMING_SNAKE_CASE						:     str					=		processor(text=lowerCAmelCase__			,			images=lowerCAmelCase__	)

										self.assertListEqual(list(inputs.keys()	)			,			['''input_ids''', '''attention_mask''', '''pixel_values''']	)

										# test if it raises when no input is passed
										with pytest.raises(lowerCAmelCase__	):
																processor()


				def 		__magic_name__(       self			:str	)		->			int:
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		self.get_image_processor()
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		self.get_tokenizer()

										__SCREAMING_SNAKE_CASE						:     str					=		CLIPProcessor(tokenizer=lowerCAmelCase__			,			image_processor=lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     List[str]					=		[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		processor.batch_decode(lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     Any					=		tokenizer.batch_decode(lowerCAmelCase__	)

										self.assertListEqual(lowerCAmelCase__			,			lowerCAmelCase__	)



				def 		__magic_name__(       self			:Any	)		->			Tuple:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		self.get_image_processor()
										__SCREAMING_SNAKE_CASE						:     Tuple					=		self.get_tokenizer()

										__SCREAMING_SNAKE_CASE						:     int					=		CLIPProcessor(tokenizer=lowerCAmelCase__			,			image_processor=lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		'''lower newer'''
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		self.prepare_image_inputs()

										__SCREAMING_SNAKE_CASE						:     int					=		processor(text=lowerCAmelCase__			,			images=lowerCAmelCase__	)

										self.assertListEqual(list(inputs.keys()	)			,			processor.model_input_names	)



import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict





@pytest.mark.parametrize(
    'files'   ,							[
        ['full:README.md', 'dataset_infos.json'],
        ['empty:README.md', 'dataset_infos.json'],
        ['dataset_infos.json'],
        ['full:README.md'],
    ]   ,							)
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							snake_case_							=    tmp_path_factory.mktemp('dset_infos_dir'      )
							if "full:README.md" in files:
														with open(dataset_infos_dir / 'README.md'   ,							'w'      ) as f:
																					f.write('---\ndataset_info:\n  dataset_size: 42\n---'      )
							if "empty:README.md" in files:
														with open(dataset_infos_dir / 'README.md'   ,							'w'      ) as f:
																					f.write(''      )
    # we want to support dataset_infos.json for backward compatibility
							if "dataset_infos.json" in files:
														with open(dataset_infos_dir / 'dataset_infos.json'   ,							'w'      ) as f:
																					f.write('{"default": {"dataset_size": 42}}'      )
							snake_case_							=    DatasetInfosDict.from_directory(UpperCamelCase__      )
							assert dataset_infos
							assert dataset_infos["default"].dataset_size == 42





@pytest.mark.parametrize(
    'dataset_info'   ,							[
        DatasetInfo(),
        DatasetInfo(
            description='foo'   ,							features=Features({'a': Value('int32'      )}      )   ,							builder_name='builder'   ,							config_name='config'   ,							version='1.0.0'   ,							splits=[{'name': 'train'}]   ,							download_size=42   ,							),
    ]   ,							)
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							snake_case_							=    str(UpperCamelCase__      )
							dataset_info.write_to_directory(UpperCamelCase__      )
							snake_case_							=    DatasetInfo.from_directory(UpperCamelCase__      )
							assert dataset_info == reloaded
							assert os.path.exists(os.path.join(UpperCamelCase__   ,							'dataset_info.json'      )      )





def 					__lowerCamelCase	( ):







							'''simple docstring'''






							snake_case_							=    DatasetInfo(
							    description='foo'   ,							citation='bar'   ,							homepage='https://foo.bar'   ,							license='CC0'   ,							features=Features({'a': Value('int32'      )}      )   ,							post_processed={}   ,							supervised_keys=()   ,							task_templates=[]   ,							builder_name='builder'   ,							config_name='config'   ,							version='1.0.0'   ,							splits=[{'name': 'train', 'num_examples': 42}]   ,							download_checksums={}   ,							download_size=1337   ,							post_processing_size=442   ,							dataset_size=1234   ,							size_in_bytes=1337 + 442 + 1234   ,							)
							snake_case_							=    dataset_info._to_yaml_dict()
							assert sorted(UpperCamelCase__      ) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML      )
							for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
														assert key in dataset_info_yaml_dict
														assert isinstance(dataset_info_yaml_dict[key]   ,							(list, dict, int, str)      )
							snake_case_							=    yaml.safe_dump(UpperCamelCase__      )
							snake_case_							=    yaml.safe_load(UpperCamelCase__      )
							assert dataset_info_yaml_dict == reloaded





def 					__lowerCamelCase	( ):







							'''simple docstring'''






							snake_case_							=    DatasetInfo()
							snake_case_							=    dataset_info._to_yaml_dict()
							assert dataset_info_yaml_dict == {}





@pytest.mark.parametrize(
    'dataset_infos_dict'   ,							[
        DatasetInfosDict(),
        DatasetInfosDict({'default': DatasetInfo()}      ),
        DatasetInfosDict({'my_config_name': DatasetInfo()}      ),
        DatasetInfosDict(
            {
                'default': DatasetInfo(
                    description='foo'   ,							features=Features({'a': Value('int32'      )}      )   ,							builder_name='builder'   ,							config_name='config'   ,							version='1.0.0'   ,							splits=[{'name': 'train'}]   ,							download_size=42   ,							)
            }      ),
        DatasetInfosDict(
            {
                'v1': DatasetInfo(dataset_size=42      ),
                'v2': DatasetInfo(dataset_size=1337      ),
            }      ),
    ]   ,							)
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							snake_case_							=    str(UpperCamelCase__      )
							dataset_infos_dict.write_to_directory(UpperCamelCase__      )
							snake_case_							=    DatasetInfosDict.from_directory(UpperCamelCase__      )

							# the config_name of the dataset_infos_dict take over the attribute
							for config_name, dataset_info in dataset_infos_dict.items():
														snake_case_							=    config_name
														# the yaml representation doesn't include fields like description or citation
														# so we just test that we can recover what we can from the yaml
														snake_case_							=    DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()      )
							assert dataset_infos_dict == reloaded

							if dataset_infos_dict:
														assert os.path.exists(os.path.join(UpperCamelCase__   ,							'README.md'      )      )




