



'''simple docstring'''






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__:   int								=			get_tests_dir("fixtures/test_sentencepiece.model")

@require_sentencepiece
class 				SCREAMING_SNAKE_CASE(							A__							,		unittest.TestCase			):







	"""simple docstring"""






	lowerCamelCase__			 =       XLMProphetNetTokenizer
	lowerCamelCase__			 =       False
	lowerCamelCase__			 =       True







	def      A	(						self			:     Dict		)			->		Any:
								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCAmelCase :       Dict 							=		XLMProphetNetTokenizer(__snake_case		,			keep_accents=__snake_case		)
								tokenizer.save_pretrained(self.tmpdirname		)







	def      A	(						self			:     Any		)			->		Tuple:
								UpperCAmelCase :       Any 							=		'''[PAD]'''
								UpperCAmelCase :       Optional[Any] 							=		0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case		)		,			__snake_case		)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case		)		,			__snake_case		)







	def      A	(						self			:     Optional[Any]		)			->		Any:
								UpperCAmelCase :       Union[str, Any] 							=		list(self.get_tokenizer().get_vocab().keys()		)

								self.assertEqual(vocab_keys[0]		,			'''[PAD]'''		)
								self.assertEqual(vocab_keys[1]		,			'''[CLS]'''		)
								self.assertEqual(vocab_keys[-1]		,			'''j'''		)
								self.assertEqual(len(__snake_case		)		,			1012		)







	def      A	(						self			:     int		)			->		Any:
								self.assertEqual(self.get_tokenizer().vocab_size		,			1012		)







	def      A	(						self			:     Tuple		)			->		int:
								UpperCAmelCase :       Optional[Any] 							=		XLMProphetNetTokenizer(__snake_case		,			keep_accents=__snake_case		)

								UpperCAmelCase :       List[Any] 							=		tokenizer.tokenize('''This is a test'''		)
								self.assertListEqual(__snake_case		,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(__snake_case		)		,			[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]		,			)

								UpperCAmelCase :       Optional[int] 							=		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
								self.assertListEqual(
								    __snake_case		,			[
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''é''',
								        '''.''',
								    ]		,			)
								UpperCAmelCase :       Union[str, Any] 							=		tokenizer.convert_tokens_to_ids(__snake_case		)
								self.assertListEqual(
								    __snake_case		,			[
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
								    ]		,			)

								UpperCAmelCase :       Tuple 							=		tokenizer.convert_ids_to_tokens(__snake_case		)
								self.assertListEqual(
								    __snake_case		,			[
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''[UNK]''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''[UNK]''',
								        '''.''',
								    ]		,			)







	@cached_property
	def      A	(						self			:     str		)			->		List[Any]:
								return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''		)







	@slow
	def      A	(						self			:     Optional[Any]		)			->		Union[str, Any]:
								UpperCAmelCase :       int 							=		'''Hello World!'''
								UpperCAmelCase :       str 							=		[35389, 6672, 49, 2]
								self.assertListEqual(__snake_case		,			self.big_tokenizer.encode(__snake_case		)		)







	@slow
	def      A	(						self			:     Dict		)			->		Union[str, Any]:
								# fmt: off
								UpperCAmelCase :       int 							=		{'''input_ids''': [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=__snake_case		,			model_name='''microsoft/xprophetnet-large-wiki100-cased'''		,			revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''		,			)












"""simple docstring"""
def 					A__				(			UpperCamelCase       ):
							A		=  generate_pascal_triangle(UpperCamelCase       )
							for row_idx in range(UpperCamelCase       ):
														# Print left spaces
														for _ in range(num_rows - row_idx - 1       ):
																					print(end=" "       )
														# Print row values
														for col_idx in range(row_idx + 1       ):
																					if col_idx != row_idx:
																												print(triangle[row_idx][col_idx]					,  end=" "       )
																					else:
																												print(triangle[row_idx][col_idx]					,  end=""       )
														print()






def 					A__				(			UpperCamelCase       ):

							if not isinstance(UpperCamelCase					,  UpperCamelCase       ):
														raise TypeError("The input value of 'num_rows' should be 'int'"       )

							if num_rows == 0:
														return []
							elif num_rows < 0:
														raise ValueError(
														    "The input value of 'num_rows' should be greater than or equal to 0"       )

							A		=  []
							for current_row_idx in range(UpperCamelCase       ):
														A		=  populate_current_row(UpperCamelCase					,  UpperCamelCase       )
														triangle.append(UpperCamelCase       )
							return triangle






def 					A__				(			UpperCamelCase					,  UpperCamelCase       ):
							A		=  [-1] * (current_row_idx + 1)
							# first and last elements of current row are equal to 1
							A,	A		=  1, 1
							for current_col_idx in range(1					,  UpperCamelCase       ):
														calculate_current_element(
														    UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase       )
							return current_row






def 					A__				(			UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase					,  ):
							A		=  triangle[current_row_idx - 1][current_col_idx - 1]
							A		=  triangle[current_row_idx - 1][current_col_idx]
							A		=  above_to_left_elt + above_to_right_elt






def 					A__				(			UpperCamelCase       ):

							if not isinstance(UpperCamelCase					,  UpperCamelCase       ):
														raise TypeError("The input value of 'num_rows' should be 'int'"       )

							if num_rows == 0:
														return []
							elif num_rows < 0:
														raise ValueError(
														    "The input value of 'num_rows' should be greater than or equal to 0"       )

							A		=  [[1]]

							for row_index in range(1					,  UpperCamelCase       ):
														A		=  [0] + result[-1] + [0]
														A		=  row_index + 1
														# Calculate the number of distinct elements in a row
														A		=  sum(divmod(UpperCamelCase					,  2       )       )
														A		=  [
														    temp_row[i - 1] + temp_row[i] for i in range(1					,  distinct_elements + 1       )
														]
														A		=  row_first_half[: (row_index + 1) // 2]
														row_second_half.reverse()
														A		=  row_first_half + row_second_half
														result.append(UpperCamelCase       )

							return result






def 					A__				(			):
							from collections.abc import Callable
							from timeit import timeit

							def benchmark_a_function(UpperCamelCase					,  UpperCamelCase       ) -> None:
														A		=  F"{func.__name__}({value})"
														A		=  timeit(F"__main__.{call}"					,  setup="import __main__"       )
														# print(f"{call:38} = {func(value)} -- {timing:.4f} seconds")
														print(F"{call:38} -- {timing:.4f} seconds"       )

							for value in range(15       ):  # (1, 7, 14):
														for func in (generate_pascal_triangle, generate_pascal_triangle_optimized):
																					benchmark_a_function(UpperCamelCase					,  UpperCamelCase       )
														print()


if __name__ == "__main__":
		import doctest

		doctest.testmod()
		benchmark()


