






'''simple docstring'''







import unittest

import numpy as np

from transformers.testing_utils import is_flaky, require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
			import torch

if is_vision_available():
			from PIL import Image

			from transformers import DonutImageProcessor

class _lowercase	(				unittest.TestCase		):




							def __init__(							self:				str			,							UpperCamelCase__:				Optional[Any]			,							UpperCamelCase__:				Any=7			,							UpperCamelCase__:				Optional[int]=3			,							UpperCamelCase__:				List[str]=18			,							UpperCamelCase__:				Union[str, Any]=30			,							UpperCamelCase__:				List[str]=400			,							UpperCamelCase__:				Any=True			,							UpperCamelCase__:				Union[str, Any]=None			,							UpperCamelCase__:				List[Any]=True			,							UpperCamelCase__:				List[Any]=False			,							UpperCamelCase__:				Tuple=True			,							UpperCamelCase__:				Tuple=True			,							UpperCamelCase__:				Any=[0.5, 0.5, 0.5]			,							UpperCamelCase__:				Optional[Any]=[0.5, 0.5, 0.5]			,							):
														lowerCamelCase__					:	int         =       parent
														lowerCamelCase__					:	Any         =       batch_size
														lowerCamelCase__					:	Optional[int]         =       num_channels
														lowerCamelCase__					:	Union[str, Any]         =       image_size
														lowerCamelCase__					:	Optional[int]         =       min_resolution
														lowerCamelCase__					:	Optional[Any]         =       max_resolution
														lowerCamelCase__					:	Union[str, Any]         =       do_resize
														lowerCamelCase__					:	Optional[Any]         =       size if size is not None else {"""height""": 18, """width""": 20}
														lowerCamelCase__					:	Dict         =       do_thumbnail
														lowerCamelCase__					:	Optional[int]         =       do_align_axis
														lowerCamelCase__					:	Any         =       do_pad
														lowerCamelCase__					:	Optional[Any]         =       do_normalize
														lowerCamelCase__					:	Union[str, Any]         =       image_mean
														lowerCamelCase__					:	Union[str, Any]         =       image_std




							def 		lowerCamelCase_				(							self:				str     ):
														return {
														    "do_resize": self.do_resize,
														    "size": self.size,
														    "do_thumbnail": self.do_thumbnail,
														    "do_align_long_axis": self.do_align_axis,
														    "do_pad": self.do_pad,
														    "do_normalize": self.do_normalize,
														    "image_mean": self.image_mean,
														    "image_std": self.image_std,
														}



@require_torch
@require_vision
class _lowercase	(				_lowercase				,      unittest.TestCase		):
							a				      =       DonutImageProcessor if is_vision_available() else None




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														lowerCamelCase__					:	Any         =       DonutImageProcessingTester(self     )




							@property
							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														return self.image_processor_tester.prepare_image_processor_dict()




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	Dict         =       self.image_processing_class(**self.image_processor_dict     )
														self.assertTrue(hasattr(UpperCamelCase__			,							"""do_resize"""     )     )
														self.assertTrue(hasattr(UpperCamelCase__			,							"""size"""     )     )
														self.assertTrue(hasattr(UpperCamelCase__			,							"""do_thumbnail"""     )     )
														self.assertTrue(hasattr(UpperCamelCase__			,							"""do_align_long_axis"""     )     )
														self.assertTrue(hasattr(UpperCamelCase__			,							"""do_pad"""     )     )
														self.assertTrue(hasattr(UpperCamelCase__			,							"""do_normalize"""     )     )
														self.assertTrue(hasattr(UpperCamelCase__			,							"""image_mean"""     )     )
														self.assertTrue(hasattr(UpperCamelCase__			,							"""image_std"""     )     )




							def 		lowerCamelCase_				(							self:				Optional[Any]     ):
														lowerCamelCase__					:	Optional[int]         =       self.image_processing_class.from_dict(self.image_processor_dict     )
														self.assertEqual(image_processor.size			,							{"""height""": 18, """width""": 20}     )

														lowerCamelCase__					:	Optional[int]         =       self.image_processing_class.from_dict(self.image_processor_dict			,							size=42     )
														self.assertEqual(image_processor.size			,							{"""height""": 42, """width""": 42}     )

														# Previous config had dimensions in (width, height) order
														lowerCamelCase__					:	List[str]         =       self.image_processing_class.from_dict(self.image_processor_dict			,							size=(42, 84)     )
														self.assertEqual(image_processor.size			,							{"""height""": 84, """width""": 42}     )




							def 		lowerCamelCase_				(							self:				List[str]     ):
														pass




							@is_flaky()
							def 		lowerCamelCase_				(							self:				Union[str, Any]     ):
														# Initialize image_processing
														lowerCamelCase__					:	str         =       self.image_processing_class(**self.image_processor_dict     )
														# create random PIL images
														lowerCamelCase__					:	Dict         =       prepare_image_inputs(self.image_processor_tester			,							equal_resolution=UpperCamelCase__     )
														for image in image_inputs:
																					self.assertIsInstance(UpperCamelCase__			,							Image.Image     )

														# Test not batched input
														lowerCamelCase__					:	Union[str, Any]         =       image_processing(image_inputs[0]			,							return_tensors="""pt"""     ).pixel_values
														self.assertEqual(
														    encoded_images.shape			,							(
														        1,
														        self.image_processor_tester.num_channels,
														        self.image_processor_tester.size["""height"""],
														        self.image_processor_tester.size["""width"""],
														    )			,							)

														# Test batched
														lowerCamelCase__					:	List[str]         =       image_processing(UpperCamelCase__			,							return_tensors="""pt"""     ).pixel_values
														self.assertEqual(
														    encoded_images.shape			,							(
														        self.image_processor_tester.batch_size,
														        self.image_processor_tester.num_channels,
														        self.image_processor_tester.size["""height"""],
														        self.image_processor_tester.size["""width"""],
														    )			,							)




							@is_flaky()
							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														# Initialize image_processing
														lowerCamelCase__					:	Dict         =       self.image_processing_class(**self.image_processor_dict     )
														# create random numpy tensors
														lowerCamelCase__					:	Dict         =       prepare_image_inputs(self.image_processor_tester			,							equal_resolution=UpperCamelCase__			,							numpify=UpperCamelCase__     )
														for image in image_inputs:
																					self.assertIsInstance(UpperCamelCase__			,							np.ndarray     )

														# Test not batched input
														lowerCamelCase__					:	List[Any]         =       image_processing(image_inputs[0]			,							return_tensors="""pt"""     ).pixel_values
														self.assertEqual(
														    encoded_images.shape			,							(
														        1,
														        self.image_processor_tester.num_channels,
														        self.image_processor_tester.size["""height"""],
														        self.image_processor_tester.size["""width"""],
														    )			,							)

														# Test batched
														lowerCamelCase__					:	Optional[Any]         =       image_processing(UpperCamelCase__			,							return_tensors="""pt"""     ).pixel_values
														self.assertEqual(
														    encoded_images.shape			,							(
														        self.image_processor_tester.batch_size,
														        self.image_processor_tester.num_channels,
														        self.image_processor_tester.size["""height"""],
														        self.image_processor_tester.size["""width"""],
														    )			,							)




							@is_flaky()
							def 		lowerCamelCase_				(							self:				Dict     ):
														# Initialize image_processing
														lowerCamelCase__					:	int         =       self.image_processing_class(**self.image_processor_dict     )
														# create random PyTorch tensors
														lowerCamelCase__					:	Optional[int]         =       prepare_image_inputs(self.image_processor_tester			,							equal_resolution=UpperCamelCase__			,							torchify=UpperCamelCase__     )
														for image in image_inputs:
																					self.assertIsInstance(UpperCamelCase__			,							torch.Tensor     )

														# Test not batched input
														lowerCamelCase__					:	Union[str, Any]         =       image_processing(image_inputs[0]			,							return_tensors="""pt"""     ).pixel_values
														self.assertEqual(
														    encoded_images.shape			,							(
														        1,
														        self.image_processor_tester.num_channels,
														        self.image_processor_tester.size["""height"""],
														        self.image_processor_tester.size["""width"""],
														    )			,							)

														# Test batched
														lowerCamelCase__					:	Tuple         =       image_processing(UpperCamelCase__			,							return_tensors="""pt"""     ).pixel_values
														self.assertEqual(
														    encoded_images.shape			,							(
														        self.image_processor_tester.batch_size,
														        self.image_processor_tester.num_channels,
														        self.image_processor_tester.size["""height"""],
														        self.image_processor_tester.size["""width"""],
														    )			,							)





"""simple docstring"""






import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class      lowerCamelCase__   (     A     ,    unittest.TestCase				):







					"""simple docstring"""


					__a	       =							None
					__a	       =							BloomTokenizerFast
					__a	       =							BloomTokenizerFast
					__a	       =							True
					__a	       =							False
					__a	       =							"""tokenizer_file"""
					__a	       =							{"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}






					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												super().setUp()
												__UpperCAmelCase   :						Any      							=					BloomTokenizerFast.from_pretrained("""bigscience/tokenizer"""  )
												tokenizer.save_pretrained(self.tmpdirname  )






					def lowerCamelCase__							(       self    : Any ,       **UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												kwargs.update(self.special_tokens_map  )
												return BloomTokenizerFast.from_pretrained(self.tmpdirname ,       **UpperCamelCase  )






					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					self.get_rust_tokenizer()

												__UpperCAmelCase   :						Optional[Any]      							=					["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
												__UpperCAmelCase   :						int      							=					[[2_175, 23_714, 73_173, 144_252, 2], [77, 132_619, 3_478, 368, 109_586, 35_433, 2]]

												__UpperCAmelCase   :						Dict      							=					tokenizer.batch_encode_plus(UpperCamelCase  )["""input_ids"""]
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )

												__UpperCAmelCase   :						int      							=					tokenizer.batch_decode(UpperCamelCase  )
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : Any=6  ):
												'''simple docstring'''






												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																										__UpperCAmelCase   :						Optional[int]      							=					self.rust_tokenizer_class.from_pretrained(UpperCamelCase ,       **UpperCamelCase  )
																										# tokenizer_r.pad_token = None # Hotfixing padding = None
																										# Simple input
																										__UpperCAmelCase   :						Dict      							=					"""This is a simple input"""
																										__UpperCAmelCase   :						str      							=					["""This is a simple input 1""", """This is a simple input 2"""]
																										__UpperCAmelCase   :						List[str]      							=					("""This is a simple input""", """This is a pair""")
																										__UpperCAmelCase   :						Dict      							=					[
																										    ("""This is a simple input 1""", """This is a simple input 2"""),
																										    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																										]

																										# Simple input tests
																										try:
																																	tokenizer_r.encode(UpperCamelCase ,       max_length=UpperCamelCase  )
																																	tokenizer_r.encode_plus(UpperCamelCase ,       max_length=UpperCamelCase  )

																																	tokenizer_r.batch_encode_plus(UpperCamelCase ,       max_length=UpperCamelCase  )
																																	tokenizer_r.encode(UpperCamelCase ,       max_length=UpperCamelCase  )
																																	tokenizer_r.batch_encode_plus(UpperCamelCase ,       max_length=UpperCamelCase  )
																										except ValueError:
																																	self.fail("""Bloom Tokenizer should be able to deal with padding"""  )

																										__UpperCAmelCase   :						Union[str, Any]      							=					None  # Hotfixing padding = None
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Simple input
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Simple input
																										self.assertRaises(
																										    UpperCamelCase ,       tokenizer_r.batch_encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length""" ,       )

																										# Pair input
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Pair input
																										self.assertRaises(UpperCamelCase ,       tokenizer_r.encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length"""  )

																										# Pair input
																										self.assertRaises(
																										    UpperCamelCase ,       tokenizer_r.batch_encode_plus ,       UpperCamelCase ,       max_length=UpperCamelCase ,       padding="""max_length""" ,       )






					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Union[str, Any]      							=					self.get_rust_tokenizer()
												__UpperCAmelCase   :						Optional[Any]      							=					load_dataset("""xnli""" ,       """all_languages""" ,       split="""test""" ,       streaming=UpperCamelCase  )

												__UpperCAmelCase   :						Optional[Any]      							=					next(iter(UpperCamelCase  )  )["""premise"""]  # pick up one data
												__UpperCAmelCase   :						Any      							=					list(sample_data.values()  )

												__UpperCAmelCase   :						Optional[Any]      							=					list(map(tokenizer.encode ,       UpperCamelCase  )  )
												__UpperCAmelCase   :						List[Any]      							=					[tokenizer.decode(UpperCamelCase ,       clean_up_tokenization_spaces=UpperCamelCase  ) for x in output_tokens]
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )






					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  ) ,       1  )
												self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  ) ,       1  )
