



import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__   			=   get_tests_dir('''fixtures/test_sentencepiece.model''')


@require_sentencepiece
@require_tokenizers
class   __lowerCAmelCase  (		UpperCAmelCase_				,  unittest.TestCase   ):
					"""simple docstring"""






					A__					:		List[str]      							=     XLNetTokenizer
					A__					:		str      							=     XLNetTokenizerFast
					A__					:		int      							=     True
					A__					:		List[str]      							=     True



					def      _a				(  self       :							Optional[int]       ):






								"""simple docstring"""


								super().setUp()

								# We have a SentencePiece fixture for testing
								A__	  =     XLNetTokenizer(_snake_case					,       keep_accents=_snake_case       )
								tokenizer.sanitize_special_tokens()
								tokenizer.save_pretrained(self.tmpdirname       )



					def      _a				(  self       :							Any       ):






								"""simple docstring"""


								A__	  =     '<s>'
								A__	  =     1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(_snake_case       )					,       _snake_case       )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(_snake_case       )					,       _snake_case       )



					def      _a				(  self       :							Tuple       ):






								"""simple docstring"""


								A__	  =     list(self.get_tokenizer().get_vocab().keys()       )

								self.assertEqual(vocab_keys[0]					,       '<unk>'       )
								self.assertEqual(vocab_keys[1]					,       '<s>'       )
								self.assertEqual(vocab_keys[-1]					,       '<eod>'       )
								self.assertEqual(len(_snake_case       )					,       10_06       )



					def      _a				(  self       :							Dict       ):






								"""simple docstring"""


								self.assertEqual(self.get_tokenizer().vocab_size					,       10_00       )



					def      _a				(  self       :							List[Any]       ):






								"""simple docstring"""


								A__	  =     XLNetTokenizer(_snake_case					,       keep_accents=_snake_case       )

								A__	  =     tokenizer.tokenize('This is a test'       )
								self.assertListEqual(_snake_case					,       ['▁This', '▁is', '▁a', '▁t', 'est']       )

								self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case       )					,       [2_85, 46, 10, 1_70, 3_82]       )

								A__	  =     tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
								self.assertListEqual(
								    _snake_case					,       [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '9',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        'é',
								        '.',
								    ]					,       )
								A__	  =     tokenizer.convert_tokens_to_ids(_snake_case       )
								self.assertListEqual(_snake_case					,       [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]       )

								A__	  =     tokenizer.convert_ids_to_tokens(_snake_case       )
								self.assertListEqual(
								    _snake_case					,       [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '<unk>',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        '<unk>',
								        '.',
								    ]					,       )



					def      _a				(  self       :							List[Any]       ):






								"""simple docstring"""


								A__	  =     XLNetTokenizer(_snake_case					,       do_lower_case=_snake_case       )
								A__	  =     tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
								self.assertListEqual(
								    _snake_case					,       [
								        SPIECE_UNDERLINE + '',
								        'i',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '9',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        'se',
								        '.',
								    ]					,       )
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )					,       ['▁he', 'll', 'o']       )



					def      _a				(  self       :							Tuple       ):






								"""simple docstring"""


								A__	  =     XLNetTokenizer(_snake_case					,       do_lower_case=_snake_case       )
								A__	  =     tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
								self.assertListEqual(
								    _snake_case					,       [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '9',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        'se',
								        '.',
								    ]					,       )



					@slow
					def      _a				(  self       :							Optional[Any]       ):






								"""simple docstring"""


								A__	  =     XLNetTokenizer.from_pretrained('xlnet-base-cased'       )

								A__	  =     tokenizer.encode('sequence builders'					,       add_special_tokens=_snake_case       )
								A__	  =     tokenizer.encode('multi-sequence build'					,       add_special_tokens=_snake_case       )

								A__	  =     tokenizer.build_inputs_with_special_tokens(_snake_case       )
								A__	  =     tokenizer.build_inputs_with_special_tokens(_snake_case					,       _snake_case       )

								assert encoded_sentence == text + [4, 3]
								assert encoded_pair == text + [4] + text_a + [4, 3]



					@slow
					def      _a				(  self       :							str       ):






								"""simple docstring"""


								A__	  =     {'input_ids': [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=_snake_case					,       model_name='xlnet-base-cased'					,       revision='c841166438c31ec7ca9a106dee7bb312b73ae511'					,       )








"""simple docstring"""

import copy
from typing import TYPE_CHECKING, Any, Mapping, Optional, OrderedDict

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging
from ..auto.configuration_auto import AutoConfig


if TYPE_CHECKING:
       from ... import PreTrainedTokenizerBase, TensorType

UpperCAmelCase__					=logging.get_logger(__name__)





class  lowerCamelCase__    (		_a     ):
      a			:  Optional[int]           =		"""vision-encoder-decoder"""
      a			:  Dict           =		True
      def __init__(		self						:						Dict   ,							**A_						:						List[Any]				):






       '''simple docstring'''


       super().__init__(**A_				)
       if "encoder" not in kwargs or "decoder" not in kwargs:
        raise ValueError(
            F'''A configuraton of type {self.model_type} cannot be instantiated because '''
            F'''not both `encoder` and `decoder` sub-configurations are passed, but only {kwargs}'''				)

       __lowercase						=						kwargs.pop("""encoder"""				)
       __lowercase						=						encoder_config.pop("""model_type"""				)
       __lowercase						=						kwargs.pop("""decoder"""				)
       __lowercase						=						decoder_config.pop("""model_type"""				)

       __lowercase						=						AutoConfig.for_model(A_   ,							**A_				)
       __lowercase						=						AutoConfig.for_model(A_   ,							**A_				)
       __lowercase						=						True
      @classmethod
      def SCREAMING_SNAKE_CASE_						(		cls						:						int   ,							A_						:						PretrainedConfig   ,							A_						:						PretrainedConfig   ,							**A_						:						Any				):






       '''simple docstring'''


       logger.info("""Setting `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config"""				)
       __lowercase						=						True
       __lowercase						=						True

       return cls(encoder=encoder_config.to_dict()   ,							decoder=decoder_config.to_dict()   ,							**A_				)






      def SCREAMING_SNAKE_CASE_						(		self						:						Union[str, Any]				):






       '''simple docstring'''


       __lowercase						=						copy.deepcopy(self.__dict__				)
       __lowercase						=						self.encoder.to_dict()
       __lowercase						=						self.decoder.to_dict()
       __lowercase						=						self.__class__.model_type
       return output






class  lowerCamelCase__    (		_a     ):
      a			:  Dict           =		version.parse("""1.11"""     )
      @property
      def SCREAMING_SNAKE_CASE_						(		self						:						Optional[int]				):






       '''simple docstring'''


       return OrderedDict(
           [
               ("""pixel_values""", {0: """batch""", 1: """num_channels""", 2: """height""", 3: """width"""}),
           ]				)
      @property
      def SCREAMING_SNAKE_CASE_						(		self						:						Tuple				):






       '''simple docstring'''


       return 1e-4






      @property
      def SCREAMING_SNAKE_CASE_						(		self						:						Any				):






       '''simple docstring'''


       return OrderedDict({"""last_hidden_state""": {0: """batch""", 1: """encoder_sequence"""}}				)






class  lowerCamelCase__    (		_a     ):
      @property
      def SCREAMING_SNAKE_CASE_						(		self						:						Dict				):






       '''simple docstring'''


       __lowercase						=						OrderedDict()
       __lowercase						=						{0: """batch""", 1: """past_decoder_sequence + sequence"""}
       __lowercase						=						{0: """batch""", 1: """past_decoder_sequence + sequence"""}
       __lowercase						=						{0: """batch""", 1: """encoder_sequence"""}

       return common_inputs






      def SCREAMING_SNAKE_CASE_						(		self						:						str   ,							A_						:						"PreTrainedTokenizerBase"   ,							A_						:						int = -1   ,							A_						:						int = -1   ,							A_						:						bool = False   ,							A_						:						Optional["TensorType"] = None   ,							):






       '''simple docstring'''


       import torch

       __lowercase						=						OrderedDict()

       __lowercase						=						super().generate_dummy_inputs(
           A_   ,							batch_size=A_   ,							seq_length=A_   ,							is_pair=A_   ,							framework=A_				)

       __lowercase				,		__lowercase						=						dummy_input["""input_ids"""].shape
       __lowercase						=						(batch, encoder_sequence, self._config.encoder_hidden_size)
       __lowercase						=						dummy_input.pop("""input_ids"""				)
       __lowercase						=						dummy_input.pop("""attention_mask"""				)
       __lowercase						=						torch.zeros(A_				)

       return common_inputs






class  lowerCamelCase__    (		_a     ):
      @property
      def SCREAMING_SNAKE_CASE_						(		self						:						str				):






       '''simple docstring'''


       pass
      def SCREAMING_SNAKE_CASE_						(		self						:						str   ,							A_						:						PretrainedConfig				):






       '''simple docstring'''


       return VisionEncoderDecoderEncoderOnnxConfig(A_				)






      def SCREAMING_SNAKE_CASE_						(		self						:						Tuple   ,							A_						:						PretrainedConfig   ,							A_						:						PretrainedConfig   ,							A_						:						str = "default"				):






       '''simple docstring'''


       __lowercase						=						encoder_config.hidden_size
       return VisionEncoderDecoderDecoderOnnxConfig(A_   ,							A_				)
