






'''simple docstring'''



import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE							=						get_tests_dir('fixtures/test_sentencepiece.model')
@require_sentencepiece
class UpperCAmelCase_       (     __A					,							unittest.TestCase					):




      """simple docstring"""




      UpperCamelCase_             =      XLMProphetNetTokenizer
      UpperCamelCase_             =      False
      UpperCamelCase_             =      True






      def 					A__   (			self      :		List[Any]		)	->	Optional[Any]:


       '''simple docstring'''

       super().setUp()

       # We have a SentencePiece fixture for testing
       lowercase       : int						=XLMProphetNetTokenizer(UpperCAmelCase  ,   keep_accents=UpperCAmelCase		)
       tokenizer.save_pretrained(self.tmpdirname		)






      def 					A__   (			self      :		Tuple		)	->	str:


       '''simple docstring'''

       lowercase       : Optional[Any]						='''[PAD]'''
       lowercase       : List[str]						=0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase		)  ,   UpperCAmelCase		)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase		)  ,   UpperCAmelCase		)






      def 					A__   (			self      :		Dict		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : List[Any]						=list(self.get_tokenizer().get_vocab().keys()		)

       self.assertEqual(vocab_keys[0]  ,   '''[PAD]'''		)
       self.assertEqual(vocab_keys[1]  ,   '''[CLS]'''		)
       self.assertEqual(vocab_keys[-1]  ,   '''j'''		)
       self.assertEqual(len(UpperCAmelCase		)  ,   1012		)






      def 					A__   (			self      :		Optional[Any]		)	->	Optional[int]:


       '''simple docstring'''

       self.assertEqual(self.get_tokenizer().vocab_size  ,   1012		)






      def 					A__   (			self      :		Any		)	->	str:


       '''simple docstring'''

       lowercase       : Dict						=XLMProphetNetTokenizer(UpperCAmelCase  ,   keep_accents=UpperCAmelCase		)

       lowercase       : Dict						=tokenizer.tokenize('''This is a test'''		)
       self.assertListEqual(UpperCAmelCase  ,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(UpperCAmelCase		)  ,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]  ,   )

       lowercase       : str						=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
       self.assertListEqual(
           UpperCAmelCase  ,   [
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''é''',
               '''.''',
           ]  ,   )
       lowercase       : List[Any]						=tokenizer.convert_tokens_to_ids(UpperCAmelCase		)
       self.assertListEqual(
           UpperCAmelCase  ,   [
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
           ]  ,   )

       lowercase       : Any						=tokenizer.convert_ids_to_tokens(UpperCAmelCase		)
       self.assertListEqual(
           UpperCAmelCase  ,   [
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''[UNK]''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''[UNK]''',
               '''.''',
           ]  ,   )






      @cached_property
      def 					A__   (			self      :		List[Any]		)	->	str:


       '''simple docstring'''

       return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''		)






      @slow
      def 					A__   (			self      :		List[Any]		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : Union[str, Any]						='''Hello World!'''
       lowercase       : Optional[Any]						=[3_5389, 6672, 49, 2]
       self.assertListEqual(UpperCAmelCase  ,   self.big_tokenizer.encode(UpperCAmelCase		)		)






      @slow
      def 					A__   (			self      :		List[Any]		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : Union[str, Any]						={'''input_ids''': [[1_1073, 8_2783, 18, 26, 8_2783, 549, 5_1540, 248, 1_7209, 1301, 217, 20, 21_5186, 1325, 147, 1_7209, 1301, 217, 20, 5_6370, 53, 12_2020, 20, 1_6477, 27, 8_7355, 4548, 20, 4728, 7_8392, 17, 15_9969, 18, 26, 2_4491, 629, 15, 538, 2_2704, 5439, 15, 2788, 2_4491, 9885, 15, 4_3534, 605, 15, 814, 1_8403, 3_3200, 29, 15, 4_3534, 2_4458, 1_2410, 111, 2_4966, 8_3669, 9637, 14_4068, 26, 850, 2_2346, 27, 147, 2_4966, 8_3669, 8_3490, 26, 3_9113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 12_2020, 11_5785, 34, 816, 1339, 4_6887, 18, 147, 5_3905, 1951, 4_2238, 4_1170, 1_7732, 834, 436, 15, 2_7523, 9_8733, 217, 147, 5542, 4981, 930, 1_7347, 16, 2], [2_0091, 629, 94, 8_2786, 58, 490, 20, 1528, 84, 5_3905, 344, 8_0592, 11_0128, 1_8822, 5267, 1306, 62, 15_2537, 308, 7997, 401, 12_4427, 549, 3_5442, 225, 109, 1_5055, 2_5748, 147, 7119, 4_3712, 34, 767, 13_5366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 6_3784, 11_9466, 17, 14_7808, 8_8214, 18, 656, 81, 32, 3296, 1_0280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=UpperCAmelCase  ,   model_name='''microsoft/xprophetnet-large-wiki100-cased'''  ,   revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''  ,   )










"""simple docstring"""






def      lowercase__ (					lowerCAmelCase__   :			str			)    ->			list[int]:



			'''simple docstring'''
			a__  :	List[str]		 =			[0 for i in range(len(lowerCAmelCase__			)			)]

			# initialize interval's left pointer and right pointer
			a__ ,	a__  :	int		 =			0, 0

			for i in range(1							,			len(lowerCAmelCase__			)			):
						# case when current index is inside the interval
						if i <= right_pointer:
									a__  :	List[Any]		 =			min(right_pointer - i + 1							,			z_result[i - left_pointer]			)
									a__  :	List[str]		 =			min_edge

						while go_next(lowerCAmelCase__							,			lowerCAmelCase__							,			lowerCAmelCase__			):
									z_result[i] += 1

						# if new index's result gives us more right interval,
						# we've to update left_pointer and right_pointer
						if i + z_result[i] - 1 > right_pointer:
									a__ ,	a__  :	str		 =			i, i + z_result[i] - 1

			return z_result






def      lowercase__ (					lowerCAmelCase__   :			int							,			lowerCAmelCase__   :			list[int]							,			lowerCAmelCase__   :			str			)    ->			bool:



			'''simple docstring'''
			return i + z_result[i] < len(lowerCAmelCase__			) and s[z_result[i]] == s[i + z_result[i]]






def      lowercase__ (					lowerCAmelCase__   :			str							,			lowerCAmelCase__   :			str			)    ->			int:



			'''simple docstring'''
			a__  :	List[Any]		 =			0
			# concatenate 'pattern' and 'input_str' and call z_function
			# with concatenated string
			a__  :	List[str]		 =			z_function(pattern + input_str			)

			for val in z_result:
						# if value is greater then length of the pattern string
						# that means this index is starting position of substring
						# which is equal to pattern string
						if val >= len(lowerCAmelCase__			):
									answer += 1

			return answer


if __name__ == "__main__":
		import doctest

		doctest.testmod()