






'''simple docstring'''







import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
			_A	:			List[Any]         ='''pt'''
elif is_tf_available():
			_A	:			Any         ='''tf'''
else:
			_A	:			List[str]         ='''jax'''

class _lowercase	(				_lowercase				,      unittest.TestCase		):
							a				      =       ByTaTokenizer
							a				      =       False




							def 		lowerCamelCase_				(							self:				str     ):
														super().setUp()
														lowerCamelCase__					:	str         =       ByTaTokenizer()
														tokenizer.save_pretrained(self.tmpdirname     )




							@cached_property
							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														return ByTaTokenizer.from_pretrained("""google/byt5-small"""     )




							def 		lowerCamelCase_				(							self:				Any			,							**UpperCamelCase__:				Tuple     ):
														return self.tokenizer_class.from_pretrained(self.tmpdirname			,							**UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Union[str, Any]			,							UpperCamelCase__:				str			,							UpperCamelCase__:				Any=False			,							UpperCamelCase__:				Union[str, Any]=20			,							UpperCamelCase__:				Optional[int]=5     ):
														# XXX The default common tokenizer tests assume that every ID is decodable on its own.
														# This assumption is invalid for ByT5 because single bytes might not be
														# valid utf-8 (byte 128 for instance).
														# Here we're overriding the smallest possible method to provide
														# a clean sequence without making the same assumption.

														lowerCamelCase__					:	List[str]         =       []
														for i in range(len(UpperCamelCase__     )     ):
																					try:
																												lowerCamelCase__					:	Tuple         =       tokenizer.decode([i]			,							clean_up_tokenization_spaces=UpperCamelCase__     )
																					except UnicodeDecodeError:
																												pass
																					toks.append((i, tok)     )

														lowerCamelCase__					:	Union[str, Any]         =       list(filter(lambda UpperCamelCase__     : re.match(R"""^[ a-zA-Z]+$"""			,							t[1]     )			,							UpperCamelCase__     )     )
														lowerCamelCase__					:	Tuple         =       list(filter(lambda UpperCamelCase__     : [t[0]] == tokenizer.encode(t[1]			,							add_special_tokens=UpperCamelCase__     )			,							UpperCamelCase__     )     )
														if max_length is not None and len(UpperCamelCase__     ) > max_length:
																					lowerCamelCase__					:	Dict         =       toks[:max_length]
														if min_length is not None and len(UpperCamelCase__     ) < min_length and len(UpperCamelCase__     ) > 0:
																					while len(UpperCamelCase__     ) < min_length:
																												lowerCamelCase__					:	Optional[Any]         =       toks + toks
        # toks_str = [t[1] for t in toks]
														lowerCamelCase__					:	Union[str, Any]         =       [t[0] for t in toks]

														# Ensure consistency
														lowerCamelCase__					:	Union[str, Any]         =       tokenizer.decode(UpperCamelCase__			,							clean_up_tokenization_spaces=UpperCamelCase__     )
														if " " not in output_txt and len(UpperCamelCase__     ) > 1:
																					lowerCamelCase__					:	Optional[int]         =       (
																					    tokenizer.decode([toks_ids[0]]			,							clean_up_tokenization_spaces=UpperCamelCase__     )
																					    + """ """
																					    + tokenizer.decode(toks_ids[1:]			,							clean_up_tokenization_spaces=UpperCamelCase__     )
																					)
														if with_prefix_space:
																					lowerCamelCase__					:	str         =       """ """ + output_txt
														lowerCamelCase__					:	List[Any]         =       tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
														return output_txt, output_ids




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	str         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Union[str, Any]         =       tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]     )
														lowerCamelCase__					:	Optional[int]         =       tokenizer(["""hi""", """I went to the gym""", """"""]     )
														self.assertListEqual(batch_with_eos_added["""input_ids"""]			,							batch_without_eos_added["""input_ids"""]     )




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	Optional[Any]         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Dict         =       """Unicode €."""
														lowerCamelCase__					:	List[Any]         =       tokenizer(UpperCamelCase__     )
														lowerCamelCase__					:	List[str]         =       [88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
														self.assertEqual(encoded["""input_ids"""]			,							UpperCamelCase__     )

														# decoding
														lowerCamelCase__					:	Tuple         =       tokenizer.decode(UpperCamelCase__     )
														self.assertEqual(UpperCamelCase__			,							"""Unicode €.</s>"""     )

														lowerCamelCase__					:	List[Any]         =       tokenizer("""e è é ê ë"""     )
														lowerCamelCase__					:	List[Any]         =       [104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
														self.assertEqual(encoded["""input_ids"""]			,							UpperCamelCase__     )
														# decoding
														lowerCamelCase__					:	str         =       tokenizer.decode(UpperCamelCase__     )
														self.assertEqual(UpperCamelCase__			,							"""e è é ê ë</s>"""     )

														# encode/decode, but with `encode` instead of `__call__`
														self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""     )     )			,							"""e è é ê ë</s>"""     )




							def 		lowerCamelCase_				(							self:				Any     ):
														lowerCamelCase__					:	int         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Tuple         =       ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
														# fmt: off
														lowerCamelCase__					:	List[str]         =       [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
														# fmt: on
														lowerCamelCase__					:	int         =       tokenizer(UpperCamelCase__			,							padding=UpperCamelCase__			,							return_tensors=UpperCamelCase__     )
														self.assertIsInstance(UpperCamelCase__			,							UpperCamelCase__     )

														if FRAMEWORK != "jax":
																					lowerCamelCase__					:	Any         =       list(batch.input_ids.numpy()[0]     )
														else:
																					lowerCamelCase__					:	str         =       list(batch.input_ids.tolist()[0]     )

														self.assertListEqual(UpperCamelCase__			,							UpperCamelCase__     )

														self.assertEqual((2, 37)			,							batch.input_ids.shape     )
														self.assertEqual((2, 37)			,							batch.attention_mask.shape     )




							def 		lowerCamelCase_				(							self:				List[Any]     ):
														lowerCamelCase__					:	List[str]         =       self.ta_base_tokenizer
														lowerCamelCase__					:	Tuple         =       ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
														lowerCamelCase__					:	Tuple         =       tokenizer(UpperCamelCase__			,							padding=UpperCamelCase__			,							return_tensors=UpperCamelCase__     )
														# check if input_ids are returned and no decoder_input_ids
														self.assertIn("""input_ids"""			,							UpperCamelCase__     )
														self.assertIn("""attention_mask"""			,							UpperCamelCase__     )
														self.assertNotIn("""decoder_input_ids"""			,							UpperCamelCase__     )
														self.assertNotIn("""decoder_attention_mask"""			,							UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Tuple     ):
														lowerCamelCase__					:	str         =       self.ta_base_tokenizer
														lowerCamelCase__					:	List[Any]         =       [
														    """Summary of the text.""",
														    """Another summary.""",
														]
														lowerCamelCase__					:	Union[str, Any]         =       tokenizer(
														    text_target=UpperCamelCase__			,							max_length=32			,							padding="""max_length"""			,							truncation=UpperCamelCase__			,							return_tensors=UpperCamelCase__     )
														self.assertEqual(32			,							targets["""input_ids"""].shape[1]     )




							def 		lowerCamelCase_				(							self:				str     ):
														lowerCamelCase__					:	Tuple         =       self.ta_base_tokenizer
														lowerCamelCase__					:	str         =       ["""A long paragraph for summarization. </s>"""]
														lowerCamelCase__					:	Optional[Any]         =       ["""Summary of the text. </s>"""]
														# fmt: off
														lowerCamelCase__					:	Tuple         =       [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
														lowerCamelCase__					:	Any         =       [86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
														# fmt: on

														lowerCamelCase__					:	Any         =       tokenizer(UpperCamelCase__			,							text_target=UpperCamelCase__     )

														self.assertEqual(UpperCamelCase__			,							batch["""input_ids"""][0]     )
														self.assertEqual(UpperCamelCase__			,							batch["""labels"""][0]     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														# safety check on max_len default value so we are sure the test works
														lowerCamelCase__					:	Any         =       self.get_tokenizers()
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												self.assertNotEqual(tokenizer.model_max_length			,							42     )

        # Now let's start the test
														lowerCamelCase__					:	Any         =       self.get_tokenizers()
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												# Isolate this from the other tests because we save additional tokens/etc
																												lowerCamelCase__					:	int         =       tempfile.mkdtemp()

																												lowerCamelCase__					:	List[str]         =       """ He is very happy, UNwant\u00E9d,running"""
																												lowerCamelCase__					:	List[Any]         =       tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												tokenizer.save_pretrained(UpperCamelCase__     )

																												lowerCamelCase__					:	List[Any]         =       tokenizer.__class__.from_pretrained(UpperCamelCase__     )
																												lowerCamelCase__					:	Optional[int]         =       after_tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												self.assertListEqual(UpperCamelCase__			,							UpperCamelCase__     )

																												shutil.rmtree(UpperCamelCase__     )

														lowerCamelCase__					:	Any         =       self.get_tokenizers(model_max_length=42     )
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												# Isolate this from the other tests because we save additional tokens/etc
																												lowerCamelCase__					:	Any         =       tempfile.mkdtemp()

																												lowerCamelCase__					:	Optional[Any]         =       """ He is very happy, UNwant\u00E9d,running"""
																												tokenizer.add_tokens(["""bim""", """bambam"""]     )
																												lowerCamelCase__					:	Optional[int]         =       tokenizer.additional_special_tokens
																												additional_special_tokens.append("""new_additional_special_token"""     )
																												tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}     )
																												lowerCamelCase__					:	List[Any]         =       tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												tokenizer.save_pretrained(UpperCamelCase__     )

																												lowerCamelCase__					:	Union[str, Any]         =       tokenizer.__class__.from_pretrained(UpperCamelCase__     )
																												lowerCamelCase__					:	int         =       after_tokenizer.encode(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__     )
																												self.assertListEqual(UpperCamelCase__			,							UpperCamelCase__     )
																												self.assertIn("""new_additional_special_token"""			,							after_tokenizer.additional_special_tokens     )
																												self.assertEqual(after_tokenizer.model_max_length			,							42     )

																												lowerCamelCase__					:	Any         =       tokenizer.__class__.from_pretrained(UpperCamelCase__			,							model_max_length=43     )
																												self.assertEqual(tokenizer.model_max_length			,							43     )

																												shutil.rmtree(UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														lowerCamelCase__					:	List[Any]         =       []
														if self.test_slow_tokenizer:
																					tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

														if self.test_rust_tokenizer:
																					tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

														for tokenizer_class, tokenizer_utils in tokenizer_list:
																					with tempfile.TemporaryDirectory() as tmp_dir:
																												tokenizer_utils.save_pretrained(UpperCamelCase__     )

																												with open(os.path.join(UpperCamelCase__			,							"""special_tokens_map.json"""     )			,							encoding="""utf-8"""     ) as json_file:
																																			lowerCamelCase__					:	Union[str, Any]         =       json.load(UpperCamelCase__     )

																												with open(os.path.join(UpperCamelCase__			,							"""tokenizer_config.json"""     )			,							encoding="""utf-8"""     ) as json_file:
																																			lowerCamelCase__					:	Optional[Any]         =       json.load(UpperCamelCase__     )

																												lowerCamelCase__					:	Optional[int]         =       [F'''<extra_id_{i}>''' for i in range(125     )]

																												lowerCamelCase__					:	int         =       added_tokens_extra_ids + [
																												    """an_additional_special_token"""
																												]
																												lowerCamelCase__					:	Optional[Any]         =       added_tokens_extra_ids + [
																												    """an_additional_special_token"""
																												]

																												with open(os.path.join(UpperCamelCase__			,							"""special_tokens_map.json"""     )			,							"""w"""			,							encoding="""utf-8"""     ) as outfile:
																																			json.dump(UpperCamelCase__			,							UpperCamelCase__     )
																												with open(os.path.join(UpperCamelCase__			,							"""tokenizer_config.json"""     )			,							"""w"""			,							encoding="""utf-8"""     ) as outfile:
																																			json.dump(UpperCamelCase__			,							UpperCamelCase__     )

																												# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																												# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																												# "special_tokens_map.json" files
																												lowerCamelCase__					:	Dict         =       tokenizer_class.from_pretrained(
																												    UpperCamelCase__			,							)
																												self.assertIn(
																												    """an_additional_special_token"""			,							tokenizer_without_change_in_init.additional_special_tokens     )
																												# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																												self.assertEqual(
																												    ["""an_additional_special_token"""]			,							tokenizer_without_change_in_init.convert_ids_to_tokens(
																												        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]     )     )			,							)

																												# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																												lowerCamelCase__					:	Optional[Any]         =       added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""			,							lstrip=UpperCamelCase__     )]
																												lowerCamelCase__					:	Any         =       tokenizer_class.from_pretrained(
																												    UpperCamelCase__			,							additional_special_tokens=UpperCamelCase__			,							)

																												self.assertIn("""a_new_additional_special_token"""			,							tokenizer.additional_special_tokens     )
																												self.assertEqual(
																												    ["""a_new_additional_special_token"""]			,							tokenizer.convert_ids_to_tokens(
																												        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]     )     )			,							)




							def 		lowerCamelCase_				(							self:				Union[str, Any]     ):
														lowerCamelCase__					:	Dict         =       []
														if self.test_slow_tokenizer:
																					tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

														if self.test_rust_tokenizer:
																					tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

														for tokenizer_class, tokenizer_utils in tokenizer_list:
																					with tempfile.TemporaryDirectory() as tmp_dir:
																												tokenizer_utils.save_pretrained(UpperCamelCase__     )

																												lowerCamelCase__					:	str         =       tokenizer_class.from_pretrained(UpperCamelCase__     )

																												self.assertTrue(tokenizer.decode([255]     ) == """"""     )




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														pass




							def 		lowerCamelCase_				(							self:				str     ):
														pass




							def 		lowerCamelCase_				(							self:				List[str]     ):
														pass




							def 		lowerCamelCase_				(							self:				Optional[int]     ):
														pass




							def 		lowerCamelCase_				(							self:				int     ):
														# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
														# and special added tokens as tokens
														lowerCamelCase__					:	Dict         =       self.get_tokenizers(fast=UpperCamelCase__			,							do_lower_case=UpperCamelCase__     )
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												lowerCamelCase__					:	Dict         =       ["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
																												lowerCamelCase__					:	Optional[int]         =       tokenizer.convert_tokens_to_string(UpperCamelCase__     )

																												self.assertIsInstance(UpperCamelCase__			,							UpperCamelCase__     )




							def 		lowerCamelCase_				(							self:				Any     ):
														lowerCamelCase__					:	str         =       self.get_tokenizers()
														for tokenizer in tokenizers:
																					with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																												lowerCamelCase__					:	str         =       [
																												    """bos_token""",
																												    """eos_token""",
																												    """unk_token""",
																												    """sep_token""",
																												    """pad_token""",
																												    """cls_token""",
																												    """mask_token""",
																												]

																												lowerCamelCase__					:	str         =       0
																												lowerCamelCase__					:	Any         =       tokenizer.convert_ids_to_tokens(
																												    UpperCamelCase__			,							skip_special_tokens=UpperCamelCase__     )

																												for attr in attributes_list:
																																			setattr(UpperCamelCase__			,							attr + """_id"""			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							UpperCamelCase__     )			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							attr + """_id"""     )			,							UpperCamelCase__     )

																																			setattr(UpperCamelCase__			,							attr + """_id"""			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							UpperCamelCase__     )			,							UpperCamelCase__     )
																																			self.assertEqual(getattr(UpperCamelCase__			,							attr + """_id"""     )			,							UpperCamelCase__     )

																												setattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""			,							[]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens"""     )			,							[]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""     )			,							[]     )

																												setattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""			,							[token_id_to_test_setters]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens"""     )			,							[token_to_test_setters]     )
																												self.assertListEqual(getattr(UpperCamelCase__			,							"""additional_special_tokens_ids"""     )			,							[token_id_to_test_setters]     )



"""simple docstring"""





# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

import torch

from ..models.auto import AutoModelForVisualQuestionAnswering, AutoProcessor
from ..utils import requires_backends
from .base import PipelineTool


if TYPE_CHECKING:
							from PIL import Image
class 						SCREAMING_SNAKE_CASE_ (						__a ):




		"""simple docstring"""





		__lowercase							:  Tuple						   =		'''dandelin/vilt-b32-finetuned-vqa'''
		__lowercase							:  str						   =		(
		    '''This is a tool that answers a question about an image. It takes an input named `image` which should be the '''
		    '''image containing the information, as well as a `question` which should be the question in English. It '''
		    '''returns a text that is the answer to the question.'''
		)
		__lowercase							:  Optional[int]						   =		'''image_qa'''
		__lowercase							:  Union[str, Any]						   =		AutoProcessor
		__lowercase							:  Optional[Any]						   =		AutoModelForVisualQuestionAnswering

		__lowercase							:  List[str]						   =		['''image''', '''text''']
		__lowercase							:  Optional[Any]						   =		['''text''']




		def __init__(     self							,  *lowerCAmelCase__							,  **lowerCAmelCase__):
				requires_backends(self							,  ["""vision"""])
				super().__init__(*lowerCAmelCase__							,  **lowerCAmelCase__)




		def 	snake_case_     (     self							,  lowerCAmelCase__							,  lowerCAmelCase__):
				return self.pre_processor(lowerCAmelCase__							,  lowerCAmelCase__							,  return_tensors="""pt""")




		def 	snake_case_     (     self							,  lowerCAmelCase__):
				with torch.no_grad():
						return self.model(**lowerCAmelCase__).logits




		def 	snake_case_     (     self							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = outputs.argmax(-1).item()
				return self.model.config.idalabel[idx]




