






"""simple docstring"""






import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_   : Tuple   			= get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase__				(			UpperCAmelCase__				,		unittest.TestCase ):






      '''simple docstring'''

      __UpperCamelCase	 =  XLNetTokenizer
      __UpperCamelCase	 =  XLNetTokenizerFast
      __UpperCamelCase	 =  True
      __UpperCamelCase	 =  True
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          super().setUp()

          # We have a SentencePiece fixture for testing
          SCREAMING_SNAKE_CASE_      :  str    						=		XLNetTokenizer(lowercase_						,	keep_accents=lowercase_)
          tokenizer.sanitize_special_tokens()
          tokenizer.save_pretrained(self.tmpdirname)
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		'''<s>'''
          SCREAMING_SNAKE_CASE_      :  Dict    						=		1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_)						,	lowercase_)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_)						,	lowercase_)
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Tuple    						=		list(self.get_tokenizer().get_vocab().keys())

          self.assertEqual(vocab_keys[0]						,	'''<unk>''')
          self.assertEqual(vocab_keys[1]						,	'''<s>''')
          self.assertEqual(vocab_keys[-1]						,	'''<eod>''')
          self.assertEqual(len(lowercase_)						,	1006)
      def _SCREAMING_SNAKE_CASE (						self     :  Tuple):




          '''simple docstring'''


          self.assertEqual(self.get_tokenizer().vocab_size						,	1000)
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		XLNetTokenizer(lowercase_						,	keep_accents=lowercase_)

          SCREAMING_SNAKE_CASE_      :  int    						=		tokenizer.tokenize('''This is a test''')
          self.assertListEqual(lowercase_						,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

          self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_)						,	[285, 46, 10, 170, 382])

          SCREAMING_SNAKE_CASE_      :  int    						=		tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
          self.assertListEqual(
              lowercase_						,	[
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''9''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''s''',
                  '''é''',
                  '''.''',
              ]						,	)
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		tokenizer.convert_tokens_to_ids(lowercase_)
          self.assertListEqual(lowercase_						,	[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4])

          SCREAMING_SNAKE_CASE_      :  List[str]    						=		tokenizer.convert_ids_to_tokens(lowercase_)
          self.assertListEqual(
              lowercase_						,	[
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''<unk>''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''s''',
                  '''<unk>''',
                  '''.''',
              ]						,	)
      def _SCREAMING_SNAKE_CASE (						self     :  Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		XLNetTokenizer(lowercase_						,	do_lower_case=lowercase_)
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
          self.assertListEqual(
              lowercase_						,	[
                  SPIECE_UNDERLINE + '''''',
                  '''i''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''9''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''se''',
                  '''.''',
              ]						,	)
          self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')						,	['''▁he''', '''ll''', '''o'''])
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  str    						=		XLNetTokenizer(lowercase_						,	do_lower_case=lowercase_)
          SCREAMING_SNAKE_CASE_      :  Dict    						=		tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
          self.assertListEqual(
              lowercase_						,	[
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''9''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''se''',
                  '''.''',
              ]						,	)
      @slow
      def _SCREAMING_SNAKE_CASE (						self     :  Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		XLNetTokenizer.from_pretrained('''xlnet-base-cased''')

          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		tokenizer.encode('''sequence builders'''						,	add_special_tokens=lowercase_)
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		tokenizer.encode('''multi-sequence build'''						,	add_special_tokens=lowercase_)

          SCREAMING_SNAKE_CASE_      :  Tuple    						=		tokenizer.build_inputs_with_special_tokens(lowercase_)
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		tokenizer.build_inputs_with_special_tokens(lowercase_						,	lowercase_)

          assert encoded_sentence == text + [4, 3]
          assert encoded_pair == text + [4] + text_a + [4, 3]



      @slow
      def _SCREAMING_SNAKE_CASE (						self     :  int):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		{'''input_ids''': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=lowercase_						,	model_name='''xlnet-base-cased'''						,	revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''						,	)










"""simple docstring"""






import warnings

from ...utils import logging
from .image_processing_deit import DeiTImageProcessor


UpperCAmelCase_   : Optional[Any]   			= logging.get_logger(__name__)




class 							lowerCAmelCase__				(			UpperCAmelCase__ ):






      '''simple docstring'''

      def __init__(						self     :  List[str]						,	*lowercase_     :  Dict						,	**lowercase_     :  Union[str, Any]):




          '''simple docstring'''


          warnings.warn(
              '''The class DeiTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'''
              ''' use DeiTImageProcessor instead.'''						,	lowercase_						,	)
          super().__init__(*lowercase_						,	**lowercase_)


