




'''simple docstring'''




import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a_      :		str							      =       get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
     import sentencepiece as sp


a_      :		Optional[Any]							      =       5
a_      :		str							      =       10


@require_sentencepiece
@require_tokenizers
class   __UpperCamelCase       ( lowerCamelCase__			,					unittest.TestCase				):
       lowercase	:    int      						=SpeechaTextTokenizer
       lowercase	:    int      						=False
       lowercase	:    List[str]      						=True






       def      lowercase__     (							self  ):



              """simple docstring"""







              super().setUp()

              lowerCamelCase_							=sp.SentencePieceProcessor()
              spm_model.Load(lowerCAmelCase  )
              lowerCamelCase_							=['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

              vocab += [spm_model.IdToPiece(id_  ) for id_ in range(len(lowerCAmelCase  )  )]
              lowerCamelCase_							=dict(zip(lowerCAmelCase,		range(len(lowerCAmelCase  )  )  )  )

              lowerCamelCase_							=Path(self.tmpdirname  )
              save_json(lowerCAmelCase,		save_dir / VOCAB_FILES_NAMES['''vocab_file''']  )
              if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                     copyfile(lowerCAmelCase,		save_dir / VOCAB_FILES_NAMES['''spm_file''']  )

              lowerCamelCase_							=SpeechaTextTokenizer.from_pretrained(self.tmpdirname  )
              tokenizer.save_pretrained(self.tmpdirname  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							='''<pad>'''
              lowerCamelCase_							=1

              self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase  ),		lowerCAmelCase  )
              self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase  ),		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=list(self.get_tokenizer().get_vocab().keys()  )

              self.assertEqual(vocab_keys[0],		'''<s>'''  )
              self.assertEqual(vocab_keys[1],		'''<pad>'''  )
              self.assertEqual(vocab_keys[-1],		'''j'''  )
              self.assertEqual(len(lowerCAmelCase  ),		1_001  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              self.assertEqual(self.get_tokenizer().vocab_size,		1_001  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=SpeechaTextTokenizer.from_pretrained(self.tmpdirname  )

              lowerCamelCase_							=tokenizer.tokenize('''This is a test'''  )
              self.assertListEqual(lowerCAmelCase,		['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']  )

              self.assertListEqual(
                  tokenizer.convert_tokens_to_ids(lowerCAmelCase  ),		[289, 50, 14, 174, 386],		)

              lowerCamelCase_							=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''  )
              self.assertListEqual(
                  lowerCAmelCase,		[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.'''],		)
              lowerCamelCase_							=tokenizer.convert_tokens_to_ids(lowerCAmelCase  )
              self.assertListEqual(lowerCAmelCase,		[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]  )

              lowerCamelCase_							=tokenizer.convert_ids_to_tokens(lowerCAmelCase  )
              self.assertListEqual(
                  lowerCAmelCase,		[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.'''],		)






       @slow
       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							={'''input_ids''': [[3_791, 797, 31, 11, 64, 797, 31, 2_429, 433, 12, 1_176, 12, 20, 786, 915, 142, 2_413, 240, 37, 3_238, 797, 31, 11, 35, 93, 915, 142, 2_413, 240, 37, 5_540, 567, 1_276, 93, 37, 610, 40, 62, 455, 657, 1_042, 123, 780, 177, 37, 309, 241, 1_298, 514, 20, 292, 2_737, 114, 2_469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3_388, 511, 459, 4, 3_555, 40, 321, 302, 705, 4, 3_388, 511, 583, 326, 5, 5, 5, 62, 3_310, 560, 177, 2_680, 217, 1_508, 32, 31, 853, 418, 64, 583, 511, 1_605, 62, 35, 93, 560, 177, 2_680, 217, 1_508, 1_521, 64, 583, 511, 519, 62, 20, 1_515, 764, 20, 149, 261, 5_625, 7_972, 20, 5_540, 567, 1_276, 93, 3_925, 1_675, 11, 15, 802, 7_972, 576, 217, 1_508, 11, 35, 93, 1_253, 2_441, 15, 289, 652, 31, 416, 321, 3_842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2_681, 1_153, 3_434, 20, 5_540, 37, 567, 126, 1_253, 2_441, 3_376, 449, 210, 431, 1_563, 177, 767, 5_540, 11, 1_203, 472, 11, 2_953, 685, 285, 364, 706, 1_153, 20, 6_799, 20, 2_869, 20, 4_464, 126, 40, 2_429, 20, 1_040, 866, 2_664, 418, 20, 318, 20, 1_726, 186, 20, 265, 522, 35, 93, 2_191, 4_634, 20, 1_040, 12, 6_799, 15, 228, 2_356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_575, 2_666, 684, 1_582, 1_176, 12, 627, 149, 619, 20, 4_902, 563, 11, 20, 149, 261, 3_420, 2_356, 174, 142, 4_714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
              # fmt: on

              self.tokenizer_integration_test_util(
                  expected_encoding=lowerCAmelCase,		model_name='''facebook/s2t-small-mustc-en-de-st''',		revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad''',		)





@require_sentencepiece
class   __UpperCamelCase       ( unittest.TestCase				):
       lowercase	:    Tuple      						='valhalla/s2t_mustc_multilinguial_medium'

       lowercase	:    Dict      						='C\'est trop cool'
       lowercase	:    str      						='Esto es genial'






       @classmethod
       def      lowercase__     (							cls  ):



              """simple docstring"""







              lowerCamelCase_							=SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name  )
              return cls






       def      lowercase__     (							self  ):



              """simple docstring"""







              self.assertEqual(self.tokenizer.lang_code_to_id['''pt'''],		4  )
              self.assertEqual(self.tokenizer.lang_code_to_id['''ru'''],		6  )
              self.assertEqual(self.tokenizer.lang_code_to_id['''it'''],		9  )
              self.assertEqual(self.tokenizer.lang_code_to_id['''de'''],		11  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              self.assertEqual(self.tokenizer.vocab_size,		10_000  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              self.assertIn(lowerCAmelCase,		self.tokenizer.all_special_ids  )
              lowerCamelCase_							=[ES_CODE, 4, 1_601, 47, 7_647, 2]
              lowerCamelCase_							=self.tokenizer.decode(lowerCAmelCase,		skip_special_tokens=lowerCAmelCase  )
              lowerCamelCase_							=self.tokenizer.decode(generated_ids[1:],		skip_special_tokens=lowerCAmelCase  )
              self.assertEqual(lowerCAmelCase,		lowerCAmelCase  )
              self.assertNotIn(self.tokenizer.eos_token,		lowerCAmelCase  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							='''fr'''
              lowerCamelCase_							=self.tokenizer(self.french_text  ).input_ids
              self.assertEqual(encoded[0],		lowerCAmelCase  )
              self.assertEqual(encoded[-1],		self.tokenizer.eos_token_id  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							='''fr'''
              self.assertListEqual(self.tokenizer.prefix_tokens,		[FR_CODE]  )

              lowerCamelCase_							='''es'''
              self.assertListEqual(self.tokenizer.prefix_tokens,		[ES_CODE]  )










from typing import Dict, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import flip_channel_order, resize, to_channel_dimension_format, to_pil_image
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_pytesseract_available, is_vision_available, logging, requires_backends


if is_vision_available():
		import PIL

# soft dependency
if is_pytesseract_available():
		import pytesseract

__lowerCAmelCase		:				List[str]       			=		logging.get_logger(__name__)




def 			__magic_name__				( A		:					Dict,  A		:					int,  A		:					Optional[int]   ):






			'''simple docstring'''







			return [
			    int(1000 * (box[0] / width)   ),
			    int(1000 * (box[1] / height)   ),
			    int(1000 * (box[2] / width)   ),
			    int(1000 * (box[3] / height)   ),
			]




def 			__magic_name__				( A		:					np.ndarray,  A		:					Optional[str],  A		:					Optional[str] = None   ):






			'''simple docstring'''







			a    =			tesseract_config if tesseract_config is not None else ""

			# apply OCR
			a    =			to_pil_image(A   )
			a    ,		a    =			pil_image.size
			a    =			pytesseract.image_to_data(A,  lang=A,  output_type="dict",  config=A   )
			a    ,		a    ,		a    ,		a    ,		a    =			data["text"], data["left"], data["top"], data["width"], data["height"]

			# filter empty words and corresponding coordinates
			a    =			[idx for idx, word in enumerate(A   ) if not word.strip()]
			a    =			[word for idx, word in enumerate(A   ) if idx not in irrelevant_indices]
			a    =			[coord for idx, coord in enumerate(A   ) if idx not in irrelevant_indices]
			a    =			[coord for idx, coord in enumerate(A   ) if idx not in irrelevant_indices]
			a    =			[coord for idx, coord in enumerate(A   ) if idx not in irrelevant_indices]
			a    =			[coord for idx, coord in enumerate(A   ) if idx not in irrelevant_indices]

			# turn coordinates into (left, top, left+width, top+height) format
			a    =			[]
			for x, y, w, h in zip(A,  A,  A,  A   ):
						a    =			[x, y, x + w, y + h]
						actual_boxes.append(A   )

			# finally, normalize the bounding boxes
			a    =			[]
			for box in actual_boxes:
						normalized_boxes.append(normalize_box(A,  A,  A   )   )

			assert len(A   ) == len(A   ), "Not as many words as there are bounding boxes"

			return words, normalized_boxes






class     snake_case__							(_UpperCamelCase   ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_       :							Any							 =					["""pixel_values"""]





				def __init__(      self				:     int					,      __lowerCamelCase				:     bool = True					,      __lowerCamelCase				:     Dict[str, int] = None					,      __lowerCamelCase				:     PILImageResampling = PILImageResampling.BILINEAR					,      __lowerCamelCase				:     bool = True					,      __lowerCamelCase				:     Optional[str] = None					,      __lowerCamelCase				:     Optional[str] = ""					,      **__lowerCamelCase				:     Tuple					,      )					->     None:
							super().__init__(**__lowerCamelCase							)
							a    =			size if size is not None else {"height": 2_24, "width": 2_24}
							a    =			get_size_dict(__lowerCamelCase							)

							a    =			do_resize
							a    =			size
							a    =			resample
							a    =			apply_ocr
							a    =			ocr_lang
							a    =			tesseract_config





				def 							__UpperCAmelCase							(      self				:     Dict					,      __lowerCamelCase				:     np.ndarray					,      __lowerCamelCase				:     Dict[str, int]					,      __lowerCamelCase				:     PILImageResampling = PILImageResampling.BILINEAR					,      __lowerCamelCase				:     Optional[Union[str, ChannelDimension]] = None					,      **__lowerCamelCase				:     Optional[int]					,      )					->     np.ndarray:
							a    =			get_size_dict(__lowerCamelCase							)
							if "height" not in size or "width" not in size:
										raise ValueError(f"""The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}"""							)
							a    =			(size["height"], size["width"])
							return resize(__lowerCamelCase					,      size=__lowerCamelCase					,      resample=__lowerCamelCase					,      data_format=__lowerCamelCase					,      **__lowerCamelCase							)





				def 							__UpperCAmelCase							(      self				:     List[Any]					,      __lowerCamelCase				:     ImageInput					,      __lowerCamelCase				:     bool = None					,      __lowerCamelCase				:     Dict[str, int] = None					,      __lowerCamelCase				:     PILImageResampling = None					,      __lowerCamelCase				:     bool = None					,      __lowerCamelCase				:     Optional[str] = None					,      __lowerCamelCase				:     Optional[str] = None					,      __lowerCamelCase				:     Optional[Union[str, TensorType]] = None					,      __lowerCamelCase				:     ChannelDimension = ChannelDimension.FIRST					,      **__lowerCamelCase				:     Optional[Any]					,      )					->     PIL.Image.Image:
							a    =			do_resize if do_resize is not None else self.do_resize
							a    =			size if size is not None else self.size
							a    =			get_size_dict(__lowerCamelCase							)
							a    =			resample if resample is not None else self.resample
							a    =			apply_ocr if apply_ocr is not None else self.apply_ocr
							a    =			ocr_lang if ocr_lang is not None else self.ocr_lang
							a    =			tesseract_config if tesseract_config is not None else self.tesseract_config

							a    =			make_list_of_images(__lowerCamelCase							)

							if not valid_images(__lowerCamelCase							):
										raise ValueError(
										    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
										    "torch.Tensor, tf.Tensor or jax.ndarray."							)

							if do_resize and size is None:
										raise ValueError("Size must be specified if do_resize is True."							)

							# All transformations expect numpy arrays.
							a    =			[to_numpy_array(__lowerCamelCase							) for image in images]

							if apply_ocr:
										requires_backends(self					,      "pytesseract"							)
										a    =			[]
										a    =			[]
										for image in images:
													a    ,		a    =			apply_tesseract(__lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase							)
													words_batch.append(__lowerCamelCase							)
													boxes_batch.append(__lowerCamelCase							)

							if do_resize:
										a    =			[self.resize(image=__lowerCamelCase					,      size=__lowerCamelCase					,      resample=__lowerCamelCase							) for image in images]

							# flip color channels from RGB to BGR (as Detectron2 requires this)
							a    =			[flip_channel_order(__lowerCamelCase							) for image in images]
							a    =			[to_channel_dimension_format(__lowerCamelCase					,      __lowerCamelCase							) for image in images]

							a    =			BatchFeature(data={"pixel_values": images}					,      tensor_type=__lowerCamelCase							)

							if apply_ocr:
										a    =			words_batch
										a    =			boxes_batch
							return data






