


"""simple docstring"""






import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase    :     Union[str, Any]   						=		get_tests_dir('fixtures/test_sentencepiece_no_bos.model')





@require_sentencepiece
@require_tokenizers
class      lowerCamelCase__   (     A     ,    unittest.TestCase				):







					"""simple docstring"""


					__a	       =							PegasusTokenizer
					__a	       =							PegasusTokenizerFast
					__a	       =							True
					__a	       =							True






					def lowerCamelCase__							(       self    : Union[str, Any]  ):
												'''simple docstring'''






												super().setUp()

												# We have a SentencePiece fixture for testing
												__UpperCAmelCase   :						List[str]      							=					PegasusTokenizer(UpperCamelCase  )
												tokenizer.save_pretrained(self.tmpdirname  )






					@cached_property
					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												return PegasusTokenizer.from_pretrained("""google/pegasus-large"""  )






					def lowerCamelCase__							(       self    : Dict ,       **UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												return PegasusTokenizer.from_pretrained(self.tmpdirname ,       **UpperCamelCase  )






					def lowerCamelCase__							(       self    : List[str] ,       UpperCamelCase    : Dict  ):
												'''simple docstring'''






												return ("This is a test", "This is a test")






					def lowerCamelCase__							(       self    : Union[str, Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					"""</s>"""
												__UpperCAmelCase   :						Dict      							=					1

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase  ) ,       UpperCamelCase  )
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase  ) ,       UpperCamelCase  )






					def lowerCamelCase__							(       self    : Dict  ):
												'''simple docstring'''






												__UpperCAmelCase   :						str      							=					list(self.get_tokenizer().get_vocab().keys()  )

												self.assertEqual(vocab_keys[0] ,       """<pad>"""  )
												self.assertEqual(vocab_keys[1] ,       """</s>"""  )
												self.assertEqual(vocab_keys[-1] ,       """v"""  )
												self.assertEqual(len(UpperCamelCase  ) ,       1_103  )






					def lowerCamelCase__							(       self    : List[str]  ):
												'''simple docstring'''






												self.assertEqual(self.get_tokenizer().vocab_size ,       1_103  )






					def lowerCamelCase__							(       self    : Tuple  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					self.rust_tokenizer_class.from_pretrained(self.tmpdirname  )
												__UpperCAmelCase   :						Any      							=					self.tokenizer_class.from_pretrained(self.tmpdirname  )
												__UpperCAmelCase   :						Union[str, Any]      							=					(
												    """Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"""
												    """ </s> <pad> <pad> <pad>"""
												)
												__UpperCAmelCase   :						Dict      							=					rust_tokenizer([raw_input_str] ,       return_tensors=UpperCamelCase ,       add_special_tokens=UpperCamelCase  ).input_ids[0]
												__UpperCAmelCase   :						Tuple      							=					py_tokenizer([raw_input_str] ,       return_tensors=UpperCamelCase ,       add_special_tokens=UpperCamelCase  ).input_ids[0]
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )






					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					self._large_tokenizer
												# <mask_1> masks whole sentence while <mask_2> masks single word
												__UpperCAmelCase   :						Optional[Any]      							=					"""<mask_1> To ensure a <mask_2> flow of bank resolutions."""
												__UpperCAmelCase   :						List[Any]      							=					[2, 413, 615, 114, 3, 1_971, 113, 1_679, 10_710, 107, 1]
												__UpperCAmelCase   :						Optional[int]      							=					tokenizer([raw_input_str] ,       return_tensors=UpperCamelCase  ).input_ids[0]
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )






					def lowerCamelCase__							(       self    : List[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Optional[int]      							=					self._large_tokenizer
												# The tracebacks for the following asserts are **better** without messages or self.assertEqual
												assert tokenizer.vocab_size == 96_103
												assert tokenizer.pad_token_id == 0
												assert tokenizer.eos_token_id == 1
												assert tokenizer.offset == 103
												assert tokenizer.unk_token_id == tokenizer.offset + 2 == 105
												assert tokenizer.unk_token == "<unk>"
												assert tokenizer.model_max_length == 1_024
												__UpperCAmelCase   :						List[Any]      							=					"""To ensure a smooth flow of bank resolutions."""
												__UpperCAmelCase   :						Any      							=					[413, 615, 114, 2_291, 1_971, 113, 1_679, 10_710, 107, 1]
												__UpperCAmelCase   :						Any      							=					tokenizer([raw_input_str] ,       return_tensors=UpperCamelCase  ).input_ids[0]
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )
												assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]  ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]






					@require_torch
					def lowerCamelCase__							(       self    : Union[str, Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					["""This is going to be way too long.""" * 150, """short example"""]
												__UpperCAmelCase   :						Optional[Any]      							=					["""not super long but more than 5 tokens""", """tiny"""]
												__UpperCAmelCase   :						Dict      							=					self._large_tokenizer(UpperCamelCase ,       padding=UpperCamelCase ,       truncation=UpperCamelCase ,       return_tensors="""pt"""  )
												__UpperCAmelCase   :						Dict      							=					self._large_tokenizer(
												    text_target=UpperCamelCase ,       max_length=5 ,       padding=UpperCamelCase ,       truncation=UpperCamelCase ,       return_tensors="""pt"""  )

												assert batch.input_ids.shape == (2, 1_024)
												assert batch.attention_mask.shape == (2, 1_024)
												assert targets["input_ids"].shape == (2, 5)
												assert len(UpperCamelCase  ) == 2  # input_ids, attention_mask.






					@slow
					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					{"""input_ids""": [[38_979, 143, 18_485, 606, 130, 26_669, 87_686, 121, 54_189, 1_129, 111, 26_669, 87_686, 121, 9_114, 14_787, 121, 13_249, 158, 592, 956, 121, 14_621, 31_576, 143, 62_613, 108, 9_688, 930, 43_430, 11_562, 62_613, 304, 108, 11_443, 897, 108, 9_314, 17_415, 63_399, 108, 11_443, 7_614, 18_316, 118, 4_284, 7_148, 12_430, 143, 1_400, 25_703, 158, 111, 4_284, 7_148, 11_772, 143, 21_297, 1_064, 158, 122, 204, 3_506, 1_754, 1_133, 14_787, 1_581, 115, 33_224, 4_482, 111, 1_355, 110, 29_173, 317, 50_833, 108, 20_147, 94_665, 111, 77_198, 107, 1], [110, 62_613, 117, 638, 112, 1_133, 121, 20_098, 1_355, 79_050, 13_872, 135, 1_596, 53_541, 1_352, 141, 13_039, 5_542, 124, 302, 518, 111, 268, 2_956, 115, 149, 4_427, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [139, 1_235, 2_799, 18_289, 17_780, 204, 109, 9_474, 1_296, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=UpperCamelCase ,       model_name="""google/bigbird-pegasus-large-arxiv""" ,       revision="""ba85d0851d708441f91440d509690f1ab6353415""" ,       )






@require_sentencepiece
@require_tokenizers
class      lowerCamelCase__   (     A     ,    unittest.TestCase				):







					"""simple docstring"""


					__a	       =							PegasusTokenizer
					__a	       =							PegasusTokenizerFast
					__a	       =							True
					__a	       =							True






					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												super().setUp()

												# We have a SentencePiece fixture for testing
												__UpperCAmelCase   :						Any      							=					PegasusTokenizer(UpperCamelCase ,       offset=0 ,       mask_token_sent=UpperCamelCase ,       mask_token="""[MASK]"""  )
												tokenizer.save_pretrained(self.tmpdirname  )






					@cached_property
					def lowerCamelCase__							(       self    : List[str]  ):
												'''simple docstring'''






												return PegasusTokenizer.from_pretrained("""google/bigbird-pegasus-large-arxiv"""  )






					def lowerCamelCase__							(       self    : Tuple ,       **UpperCamelCase    : List[Any]  ):
												'''simple docstring'''






												return PegasusTokenizer.from_pretrained(self.tmpdirname ,       **UpperCamelCase  )






					def lowerCamelCase__							(       self    : List[Any] ,       UpperCamelCase    : int  ):
												'''simple docstring'''






												return ("This is a test", "This is a test")






					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					self.rust_tokenizer_class.from_pretrained(self.tmpdirname  )
												__UpperCAmelCase   :						int      							=					self.tokenizer_class.from_pretrained(self.tmpdirname  )
												__UpperCAmelCase   :						Optional[Any]      							=					(
												    """Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"""
												    """ <pad> <pad> <pad>"""
												)
												__UpperCAmelCase   :						Optional[Any]      							=					rust_tokenizer([raw_input_str] ,       return_tensors=UpperCamelCase ,       add_special_tokens=UpperCamelCase  ).input_ids[0]
												__UpperCAmelCase   :						List[str]      							=					py_tokenizer([raw_input_str] ,       return_tensors=UpperCamelCase ,       add_special_tokens=UpperCamelCase  ).input_ids[0]
												self.assertListEqual(UpperCamelCase ,       UpperCamelCase  )






					@require_torch
					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					["""This is going to be way too long.""" * 1_000, """short example"""]
												__UpperCAmelCase   :						str      							=					["""not super long but more than 5 tokens""", """tiny"""]
												__UpperCAmelCase   :						int      							=					self._large_tokenizer(UpperCamelCase ,       padding=UpperCamelCase ,       truncation=UpperCamelCase ,       return_tensors="""pt"""  )
												__UpperCAmelCase   :						List[str]      							=					self._large_tokenizer(
												    text_target=UpperCamelCase ,       max_length=5 ,       padding=UpperCamelCase ,       truncation=UpperCamelCase ,       return_tensors="""pt"""  )

												assert batch.input_ids.shape == (2, 4_096)
												assert batch.attention_mask.shape == (2, 4_096)
												assert targets["input_ids"].shape == (2, 5)
												assert len(UpperCamelCase  ) == 2  # input_ids, attention_mask.






					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						str      							=					(
												    """This is an example string that is used to test the original TF implementation against the HF"""
												    """ implementation"""
												)

												__UpperCAmelCase   :						Dict      							=					self._large_tokenizer(UpperCamelCase  ).input_ids

												self.assertListEqual(
												    UpperCamelCase ,       [182, 117, 142, 587, 4_211, 120, 117, 263, 112, 804, 109, 856, 25_016, 3_137, 464, 109, 26_955, 3_137, 1] ,       )




"""simple docstring"""






import os
import sys
from contextlib import contextmanager


# Windows only
if os.name == "nt":
							import ctypes
							import msvcrt  # noqa





							class      lowerCamelCase__   (     ctypes.Structure				):







												"""simple docstring"""


												__a	       =							[("""size""", ctypes.c_int), ("""visible""", ctypes.c_byte)]
def      lowerCamelCase       (     )		->       Optional[int]:

							'''simple docstring'''



							if os.name == "nt":
														__UpperCAmelCase   :						Dict      							=					CursorInfo()
														__UpperCAmelCase   :						Any      							=					ctypes.windll.kernelaa.GetStdHandle(-1_1							)
														ctypes.windll.kernelaa.GetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
														__UpperCAmelCase   :						Tuple      							=					False
														ctypes.windll.kernelaa.SetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
							elif os.name == "posix":
														sys.stdout.write("""\033[?25l"""							)
														sys.stdout.flush()
def      lowerCamelCase       (     )		->       Optional[int]:

							'''simple docstring'''



							if os.name == "nt":
														__UpperCAmelCase   :						str      							=					CursorInfo()
														__UpperCAmelCase   :						int      							=					ctypes.windll.kernelaa.GetStdHandle(-1_1							)
														ctypes.windll.kernelaa.GetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
														__UpperCAmelCase   :						Union[str, Any]      							=					True
														ctypes.windll.kernelaa.SetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
							elif os.name == "posix":
														sys.stdout.write("""\033[?25h"""							)
														sys.stdout.flush()
@contextmanager
def      lowerCamelCase       (     )		->       str:

							'''simple docstring'''



							try:
														hide_cursor()
														yield
							finally:
														show_cursor()
