


'''simple docstring'''
import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
		from PIL import Image

		from transformers import OwlViTImageProcessor, OwlViTProcessor
@require_vision
class    lowercase	(							unittest.TestCase					):

		"""simple docstring"""

		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :Optional[Any]												= tempfile.mkdtemp()

							# fmt: off
							UpperCamelCase__   :int												= ['''''', '''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
							# fmt: on
							UpperCamelCase__   :Tuple												= dict(zip(UpperCamelCase_    ,   range(len(UpperCamelCase_							)							)							)							)
							UpperCamelCase__   :Union[str, Any]												= ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''', '''''']
							UpperCamelCase__   :Tuple												= {'''unk_token''': '''<unk>'''}

							UpperCamelCase__   :Optional[int]												= os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['''vocab_file''']							)
							UpperCamelCase__   :List[Any]												= os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['''merges_file''']							)
							with open(self.vocab_file    ,   '''w'''    ,   encoding='''utf-8'''							) as fp:
												fp.write(json.dumps(UpperCamelCase_							) + '''\n'''							)
							with open(self.merges_file    ,   '''w'''    ,   encoding='''utf-8'''							) as fp:
												fp.write('''\n'''.join(UpperCamelCase_							)							)

							UpperCamelCase__   :List[Any]												= {
							    '''do_resize''': True,
							    '''size''': 20,
							    '''do_center_crop''': True,
							    '''crop_size''': 18,
							    '''do_normalize''': True,
							    '''image_mean''': [0.48145466, 0.4578275, 0.40821073],
							    '''image_std''': [0.26862954, 0.26130258, 0.27577711],
							}
							UpperCamelCase__   :str												= os.path.join(self.tmpdirname    ,   UpperCamelCase_							)
							with open(self.image_processor_file    ,   '''w'''    ,   encoding='''utf-8'''							) as fp:
												json.dump(UpperCamelCase_    ,   UpperCamelCase_							)




		def     lowerCAmelCase__				(		self    ,   **UpperCamelCase_							):

							'''simple docstring'''


							return CLIPTokenizer.from_pretrained(self.tmpdirname    ,   pad_token='''!'''    ,   **UpperCamelCase_							)




		def     lowerCAmelCase__				(		self    ,   **UpperCamelCase_							):

							'''simple docstring'''


							return CLIPTokenizerFast.from_pretrained(self.tmpdirname    ,   pad_token='''!'''    ,   **UpperCamelCase_							)




		def     lowerCAmelCase__				(		self    ,   **UpperCamelCase_							):

							'''simple docstring'''


							return OwlViTImageProcessor.from_pretrained(self.tmpdirname    ,   **UpperCamelCase_							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							shutil.rmtree(self.tmpdirname							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :int												= [np.random.randint(255    ,   size=(3, 30, 400)    ,   dtype=np.uinta							)]

							UpperCamelCase__   :int												= [Image.fromarray(np.moveaxis(UpperCamelCase_    ,   0    ,   -1							)							) for x in image_inputs]

							return image_inputs




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :List[Any]												= self.get_tokenizer()
							UpperCamelCase__   :str												= self.get_rust_tokenizer()
							UpperCamelCase__   :Tuple												= self.get_image_processor()

							UpperCamelCase__   :Dict												= OwlViTProcessor(tokenizer=UpperCamelCase_    ,   image_processor=UpperCamelCase_							)
							processor_slow.save_pretrained(self.tmpdirname							)
							UpperCamelCase__   :Union[str, Any]												= OwlViTProcessor.from_pretrained(self.tmpdirname    ,   use_fast=UpperCamelCase_							)

							UpperCamelCase__   :List[Any]												= OwlViTProcessor(tokenizer=UpperCamelCase_    ,   image_processor=UpperCamelCase_							)
							processor_fast.save_pretrained(self.tmpdirname							)
							UpperCamelCase__   :Union[str, Any]												= OwlViTProcessor.from_pretrained(self.tmpdirname							)

							self.assertEqual(processor_slow.tokenizer.get_vocab()    ,   tokenizer_slow.get_vocab()							)
							self.assertEqual(processor_fast.tokenizer.get_vocab()    ,   tokenizer_fast.get_vocab()							)
							self.assertEqual(tokenizer_slow.get_vocab()    ,   tokenizer_fast.get_vocab()							)
							self.assertIsInstance(processor_slow.tokenizer    ,   UpperCamelCase_							)
							self.assertIsInstance(processor_fast.tokenizer    ,   UpperCamelCase_							)

							self.assertEqual(processor_slow.image_processor.to_json_string()    ,   image_processor.to_json_string()							)
							self.assertEqual(processor_fast.image_processor.to_json_string()    ,   image_processor.to_json_string()							)
							self.assertIsInstance(processor_slow.image_processor    ,   UpperCamelCase_							)
							self.assertIsInstance(processor_fast.image_processor    ,   UpperCamelCase_							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :str												= OwlViTProcessor(tokenizer=self.get_tokenizer()    ,   image_processor=self.get_image_processor()							)
							processor.save_pretrained(self.tmpdirname							)

							UpperCamelCase__   :Union[str, Any]												= self.get_tokenizer(bos_token='''(BOS)'''    ,   eos_token='''(EOS)'''							)
							UpperCamelCase__   :str												= self.get_image_processor(do_normalize=UpperCamelCase_							)

							UpperCamelCase__   :Union[str, Any]												= OwlViTProcessor.from_pretrained(
							    self.tmpdirname    ,   bos_token='''(BOS)'''    ,   eos_token='''(EOS)'''    ,   do_normalize=UpperCamelCase_							)

							self.assertEqual(processor.tokenizer.get_vocab()    ,   tokenizer_add_kwargs.get_vocab()							)
							self.assertIsInstance(processor.tokenizer    ,   UpperCamelCase_							)

							self.assertEqual(processor.image_processor.to_json_string()    ,   image_processor_add_kwargs.to_json_string()							)
							self.assertIsInstance(processor.image_processor    ,   UpperCamelCase_							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :Dict												= self.get_image_processor()
							UpperCamelCase__   :Tuple												= self.get_tokenizer()

							UpperCamelCase__   :Tuple												= OwlViTProcessor(tokenizer=UpperCamelCase_    ,   image_processor=UpperCamelCase_							)

							UpperCamelCase__   :Tuple												= self.prepare_image_inputs()

							UpperCamelCase__   :Optional[int]												= image_processor(UpperCamelCase_    ,   return_tensors='''np'''							)
							UpperCamelCase__   :str												= processor(images=UpperCamelCase_    ,   return_tensors='''np'''							)

							for key in input_image_proc.keys():
												self.assertAlmostEqual(input_image_proc[key].sum()    ,   input_processor[key].sum()    ,   delta=1e-2							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :int												= self.get_image_processor()
							UpperCamelCase__   :Union[str, Any]												= self.get_tokenizer()

							UpperCamelCase__   :List[str]												= OwlViTProcessor(tokenizer=UpperCamelCase_    ,   image_processor=UpperCamelCase_							)

							UpperCamelCase__   :Any												= '''lower newer'''

							UpperCamelCase__   :List[str]												= processor(text=UpperCamelCase_    ,   return_tensors='''np'''							)

							UpperCamelCase__   :Union[str, Any]												= tokenizer(UpperCamelCase_    ,   return_tensors='''np'''							)

							for key in encoded_tok.keys():
												self.assertListEqual(encoded_tok[key][0].tolist()    ,   encoded_processor[key][0].tolist()							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :Union[str, Any]												= self.get_image_processor()
							UpperCamelCase__   :Optional[Any]												= self.get_tokenizer()

							UpperCamelCase__   :List[Any]												= OwlViTProcessor(tokenizer=UpperCamelCase_    ,   image_processor=UpperCamelCase_							)

							UpperCamelCase__   :int												= '''lower newer'''
							UpperCamelCase__   :List[str]												= self.prepare_image_inputs()

							UpperCamelCase__   :Optional[int]												= processor(text=UpperCamelCase_    ,   images=UpperCamelCase_							)

							self.assertListEqual(list(inputs.keys()							)    ,   ['''input_ids''', '''attention_mask''', '''pixel_values''']							)

							# test if it raises when no input is passed
							with pytest.raises(UpperCamelCase_							):
												processor()




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :Optional[int]												= '''google/owlvit-base-patch32'''
							UpperCamelCase__   :Any												= OwlViTProcessor.from_pretrained(UpperCamelCase_							)

							UpperCamelCase__   :Union[str, Any]												= ['''cat''', '''nasa badge''']
							UpperCamelCase__   :List[Any]												= processor(text=UpperCamelCase_							)

							UpperCamelCase__   :List[str]												= 16
							self.assertListEqual(list(inputs.keys()							)    ,   ['''input_ids''', '''attention_mask''']							)
							self.assertEqual(inputs['''input_ids'''].shape    ,   (2, seq_length)							)

							# test if it raises when no input is passed
							with pytest.raises(UpperCamelCase_							):
												processor()




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :List[str]												= '''google/owlvit-base-patch32'''
							UpperCamelCase__   :Optional[Any]												= OwlViTProcessor.from_pretrained(UpperCamelCase_							)

							UpperCamelCase__   :Any												= [['''cat''', '''nasa badge'''], ['''person''']]
							UpperCamelCase__   :List[str]												= processor(text=UpperCamelCase_							)

							UpperCamelCase__   :int												= 16
							UpperCamelCase__   :Optional[int]												= len(UpperCamelCase_							)
							UpperCamelCase__   :int												= max([len(UpperCamelCase_							) for texts in input_texts]							)

							self.assertListEqual(list(inputs.keys()							)    ,   ['''input_ids''', '''attention_mask''']							)
							self.assertEqual(inputs['''input_ids'''].shape    ,   (batch_size * num_max_text_queries, seq_length)							)

							# test if it raises when no input is passed
							with pytest.raises(UpperCamelCase_							):
												processor()




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :str												= '''google/owlvit-base-patch32'''
							UpperCamelCase__   :List[Any]												= OwlViTProcessor.from_pretrained(UpperCamelCase_							)

							UpperCamelCase__   :str												= ['''cat''', '''nasa badge''']
							UpperCamelCase__   :Union[str, Any]												= processor(text=UpperCamelCase_							)

							UpperCamelCase__   :Dict												= 16
							UpperCamelCase__   :str												= inputs['''input_ids''']
							UpperCamelCase__   :Union[str, Any]												= [
							    [49406, 2368, 49407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							    [49406, 6841, 11301, 49407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							]

							self.assertListEqual(list(inputs.keys()							)    ,   ['''input_ids''', '''attention_mask''']							)
							self.assertEqual(inputs['''input_ids'''].shape    ,   (2, seq_length)							)
							self.assertListEqual(list(input_ids[0]							)    ,   predicted_ids[0]							)
							self.assertListEqual(list(input_ids[1]							)    ,   predicted_ids[1]							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :int												= self.get_image_processor()
							UpperCamelCase__   :Any												= self.get_tokenizer()

							UpperCamelCase__   :List[str]												= OwlViTProcessor(tokenizer=UpperCamelCase_    ,   image_processor=UpperCamelCase_							)

							UpperCamelCase__   :Optional[int]												= self.prepare_image_inputs()
							UpperCamelCase__   :Union[str, Any]												= self.prepare_image_inputs()

							UpperCamelCase__   :Any												= processor(images=UpperCamelCase_    ,   query_images=UpperCamelCase_							)

							self.assertListEqual(list(inputs.keys()							)    ,   ['''query_pixel_values''', '''pixel_values''']							)

							# test if it raises when no input is passed
							with pytest.raises(UpperCamelCase_							):
												processor()




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :List[Any]												= self.get_image_processor()
							UpperCamelCase__   :Dict												= self.get_tokenizer()

							UpperCamelCase__   :Dict												= OwlViTProcessor(tokenizer=UpperCamelCase_    ,   image_processor=UpperCamelCase_							)

							UpperCamelCase__   :Union[str, Any]												= [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

							UpperCamelCase__   :Tuple												= processor.batch_decode(UpperCamelCase_							)
							UpperCamelCase__   :Union[str, Any]												= tokenizer.batch_decode(UpperCamelCase_							)

							self.assertListEqual(UpperCamelCase_    ,   UpperCamelCase_							)



"""simple docstring"""




import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam
class 					lowerCAmelCase	(					datasets.BeamBasedBuilder    ):




		'''simple docstring'''







		def 							__A						(	self				)     ->							Dict:
								return datasets.DatasetInfo(
								    features=datasets.Features({'content': datasets.Value('string'				)}				)						,       supervised_keys=lowerCAmelCase__						,       )







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__				)     ->							str:
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,       gen_kwargs={'examples': get_test_dummy_examples()}				)]







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__				)     ->							Optional[int]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(lowerCAmelCase__				)





class 					lowerCAmelCase	(					datasets.BeamBasedBuilder    ):




		'''simple docstring'''







		def 							__A						(	self				)     ->							int:
								return datasets.DatasetInfo(
								    features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'				)}				)}				)						,       supervised_keys=lowerCAmelCase__						,       )







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__				)     ->							Dict:
								return [
								    datasets.SplitGenerator(name=datasets.Split.TRAIN						,       gen_kwargs={'examples': get_test_nested_examples()}				)
								]







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__				)     ->							Optional[Any]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(lowerCAmelCase__				)





def     lowercase			()   ->						str:
						return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar']  )]


def     lowercase			()   ->						Optional[Any]:
						return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar']  )]


class 					lowerCAmelCase	(					lowerCamelCase_    ):




		'''simple docstring'''







		@require_beam
		def 							__A						(	self				)     ->							Optional[int]:
								SCREAMING_SNAKE_CASE													=					len(get_test_dummy_examples()				)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
														SCREAMING_SNAKE_CASE													=					DummyBeamDataset(cache_dir=lowerCAmelCase__						,       beam_runner='DirectRunner'				)
														builder.download_and_prepare()
														self.assertTrue(
														    os.path.exists(
														        os.path.join(lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       F'{builder.name}-train.arrow'				)				)				)
														self.assertDictEqual(builder.info.features						,       datasets.Features({'content': datasets.Value('string'				)}				)				)
														SCREAMING_SNAKE_CASE													=					builder.as_dataset()
														self.assertEqual(dset['train'].num_rows						,       lowerCAmelCase__				)
														self.assertEqual(dset['train'].info.splits['train'].num_examples						,       lowerCAmelCase__				)
														self.assertDictEqual(dset['train'][0]						,       get_test_dummy_examples()[0][1]				)
														self.assertDictEqual(
														    dset['train'][expected_num_examples - 1]						,       get_test_dummy_examples()[expected_num_examples - 1][1]				)
														self.assertTrue(
														    os.path.exists(os.path.join(lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       'dataset_info.json'				)				)				)
														del dset







		@require_beam
		def 							__A						(	self				)     ->							int:
								import apache_beam as beam

								SCREAMING_SNAKE_CASE													=					beam.io.parquetio.WriteToParquet

								SCREAMING_SNAKE_CASE													=					len(get_test_dummy_examples()				)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
														SCREAMING_SNAKE_CASE													=					DummyBeamDataset(cache_dir=lowerCAmelCase__						,       beam_runner='DirectRunner'				)
														with patch('apache_beam.io.parquetio.WriteToParquet'				) as write_parquet_mock:
																				SCREAMING_SNAKE_CASE													=					partial(lowerCAmelCase__						,       num_shards=2				)
																				builder.download_and_prepare()
														self.assertTrue(
														    os.path.exists(
														        os.path.join(
														            lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       F'{builder.name}-train-00000-of-00002.arrow'				)				)				)
														self.assertTrue(
														    os.path.exists(
														        os.path.join(
														            lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       F'{builder.name}-train-00000-of-00002.arrow'				)				)				)
														self.assertDictEqual(builder.info.features						,       datasets.Features({'content': datasets.Value('string'				)}				)				)
														SCREAMING_SNAKE_CASE													=					builder.as_dataset()
														self.assertEqual(dset['train'].num_rows						,       lowerCAmelCase__				)
														self.assertEqual(dset['train'].info.splits['train'].num_examples						,       lowerCAmelCase__				)
														# Order is not preserved when sharding, so we just check that all the elements are there
														self.assertListEqual(sorted(dset['train']['content']				)						,       sorted(['foo', 'bar', 'foobar']				)				)
														self.assertTrue(
														    os.path.exists(os.path.join(lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       'dataset_info.json'				)				)				)
														del dset







		@require_beam
		def 							__A						(	self				)     ->							Optional[Any]:
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
														SCREAMING_SNAKE_CASE													=					DummyBeamDataset(cache_dir=lowerCAmelCase__				)
														self.assertRaises(datasets.builder.MissingBeamOptions						,       builder.download_and_prepare				)







		@require_beam
		def 							__A						(	self				)     ->							int:
								SCREAMING_SNAKE_CASE													=					len(get_test_nested_examples()				)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
														SCREAMING_SNAKE_CASE													=					NestedBeamDataset(cache_dir=lowerCAmelCase__						,       beam_runner='DirectRunner'				)
														builder.download_and_prepare()
														self.assertTrue(
														    os.path.exists(
														        os.path.join(lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       F'{builder.name}-train.arrow'				)				)				)
														self.assertDictEqual(
														    builder.info.features						,       datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'				)}				)}				)				)
														SCREAMING_SNAKE_CASE													=					builder.as_dataset()
														self.assertEqual(dset['train'].num_rows						,       lowerCAmelCase__				)
														self.assertEqual(dset['train'].info.splits['train'].num_examples						,       lowerCAmelCase__				)
														self.assertDictEqual(dset['train'][0]						,       get_test_nested_examples()[0][1]				)
														self.assertDictEqual(
														    dset['train'][expected_num_examples - 1]						,       get_test_nested_examples()[expected_num_examples - 1][1]				)
														self.assertTrue(
														    os.path.exists(os.path.join(lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       'dataset_info.json'				)				)				)
														del dset


