"""simple docstring"""




from __future__ import annotations

import random
import unittest

from transformers import TransfoXLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFTransfoXLForSequenceClassification,
         TFTransfoXLLMHeadModel,
         TFTransfoXLModel,
     )
class 						__A							:






     def __init__(		self				,     a__				,     ):
       _lowerCAmelCase						:       Optional[Any]  			=     parent
       _lowerCAmelCase						:       Optional[Any]  			=     13
       _lowerCAmelCase						:       int  			=     7
       _lowerCAmelCase						:       Dict  			=     30
       _lowerCAmelCase						:       Optional[Any]  			=     self.seq_length + self.mem_len
       _lowerCAmelCase						:       Tuple  			=     15
       _lowerCAmelCase						:       Any  			=     True
       _lowerCAmelCase						:       Dict  			=     True
       _lowerCAmelCase						:       Tuple  			=     99
       _lowerCAmelCase						:       str  			=     [10, 50, 80]
       _lowerCAmelCase						:       Tuple  			=     32
       _lowerCAmelCase						:       List[str]  			=     32
       _lowerCAmelCase						:       Optional[Any]  			=     4
       _lowerCAmelCase						:       int  			=     8
       _lowerCAmelCase						:       List[str]  			=     128
       _lowerCAmelCase						:       Optional[int]  			=     2
       _lowerCAmelCase						:       Optional[Any]  			=     2
       _lowerCAmelCase						:       Union[str, Any]  			=     None
       _lowerCAmelCase						:       Union[str, Any]  			=     1
       _lowerCAmelCase						:       Union[str, Any]  			=     0
       _lowerCAmelCase						:       List[str]  			=     3
       _lowerCAmelCase						:       List[str]  			=     self.vocab_size - 1
       _lowerCAmelCase						:       Tuple  			=     0.0_1






     def     __A  (		self  ):
       _lowerCAmelCase						:       int  			=     ids_tensor([self.batch_size, self.seq_length]				,     self.vocab_size  )
       _lowerCAmelCase						:       Dict  			=     ids_tensor([self.batch_size, self.seq_length]				,     self.vocab_size  )

       _lowerCAmelCase						:       Union[str, Any]  			=     None
       if self.use_labels:
         _lowerCAmelCase						:       int  			=     ids_tensor([self.batch_size, self.seq_length]				,     self.vocab_size  )

       _lowerCAmelCase						:       Optional[int]  			=     TransfoXLConfig(
           vocab_size=self.vocab_size				,     mem_len=self.mem_len				,     clamp_len=self.clamp_len				,     cutoffs=self.cutoffs				,     d_model=self.hidden_size				,     d_embed=self.d_embed				,     n_head=self.num_attention_heads				,     d_head=self.d_head				,     d_inner=self.d_inner				,     div_val=self.div_val				,     n_layer=self.num_hidden_layers				,     eos_token_id=self.eos_token_id				,     pad_token_id=self.vocab_size - 1				,     init_range=self.init_range				,     num_labels=self.num_labels				,     )

       return (config, input_ids_a, input_ids_a, lm_labels)






     def     __A  (		self  ):
       random.seed(self.seed  )
       tf.random.set_seed(self.seed  )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__  ):
       _lowerCAmelCase						:       Tuple  			=     TFTransfoXLModel(lowerCamelCase__  )

       _lowerCAmelCase			,	_lowerCAmelCase						:       int  			=     model(lowerCamelCase__  ).to_tuple()

       _lowerCAmelCase						:       Dict  			=     {"""input_ids""": input_ids_a, """mems""": mems_a}

       _lowerCAmelCase			,	_lowerCAmelCase						:       int  			=     model(lowerCamelCase__  ).to_tuple()

       self.parent.assertEqual(hidden_states_a.shape				,     (self.batch_size, self.seq_length, self.hidden_size)  )
       self.parent.assertEqual(hidden_states_a.shape				,     (self.batch_size, self.seq_length, self.hidden_size)  )
       self.parent.assertListEqual(
           [mem.shape for mem in mems_a]				,     [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers				,     )
       self.parent.assertListEqual(
           [mem.shape for mem in mems_a]				,     [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers				,     )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__  ):
       _lowerCAmelCase						:       str  			=     TFTransfoXLLMHeadModel(lowerCamelCase__  )

       _lowerCAmelCase			,	_lowerCAmelCase						:       str  			=     model(lowerCamelCase__  ).to_tuple()

       _lowerCAmelCase						:       List[Any]  			=     {"""input_ids""": input_ids_a, """labels""": lm_labels}
       _lowerCAmelCase			,	_lowerCAmelCase						:       Any  			=     model(lowerCamelCase__  ).to_tuple()

       _lowerCAmelCase			,	_lowerCAmelCase						:       Optional[int]  			=     model([input_ids_a, mems_a]  ).to_tuple()

       _lowerCAmelCase						:       Optional[Any]  			=     {"""input_ids""": input_ids_a, """mems""": mems_a, """labels""": lm_labels}

       _lowerCAmelCase			,	_lowerCAmelCase						:       int  			=     model(lowerCamelCase__  ).to_tuple()

       self.parent.assertEqual(lm_logits_a.shape				,     (self.batch_size, self.seq_length, self.vocab_size)  )
       self.parent.assertListEqual(
           [mem.shape for mem in mems_a]				,     [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers				,     )

       self.parent.assertEqual(lm_logits_a.shape				,     (self.batch_size, self.seq_length, self.vocab_size)  )
       self.parent.assertListEqual(
           [mem.shape for mem in mems_a]				,     [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers				,     )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__  ):
       _lowerCAmelCase						:       List[str]  			=     TFTransfoXLForSequenceClassification(lowerCamelCase__  )
       _lowerCAmelCase						:       int  			=     model(lowerCamelCase__  )
       self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.num_labels)  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Tuple  			=     self.prepare_config_and_inputs()
       ((_lowerCAmelCase)			,	(_lowerCAmelCase)			,	(_lowerCAmelCase)			,	(_lowerCAmelCase))						:       int  			=     config_and_inputs
       _lowerCAmelCase						:       Optional[int]  			=     {"""input_ids""": input_ids_a}
       return config, inputs_dict







@require_tf
class 						__A							( __UpperCAmelCase    ,	__UpperCAmelCase    ,	unittest.TestCase		):
     _UpperCamelCase							:			Dict							=		(
         (TFTransfoXLModel, TFTransfoXLLMHeadModel, TFTransfoXLForSequenceClassification) if is_tf_available() else ()
     )
     _UpperCamelCase							:			Any							=		() if is_tf_available() else ()
     _UpperCamelCase							:			Any							=		(
         {
             "feature-extraction": TFTransfoXLModel,
             "text-classification": TFTransfoXLForSequenceClassification,
             "text-generation": TFTransfoXLLMHeadModel,
             "zero-shot": TFTransfoXLForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     # TODO: add this test when TFTransfoXLLMHead has a linear output layer implemented
     _UpperCamelCase							:			Any							=		False
     _UpperCamelCase							:			Optional[Any]							=		False
     _UpperCamelCase							:			Union[str, Any]							=		False
     _UpperCamelCase							:			str							=		False






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__				,     a__  ):
       if pipeline_test_casse_name == "TextGenerationPipelineTests":
         # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`.
         # `TransfoXLConfig` was never used in pipeline tests: cannot create a simple
         # tokenizer.
         return True

       return False






     def     __A  (		self  ):
       _lowerCAmelCase						:       Dict  			=     TFTransfoXLModelTester(self  )
       _lowerCAmelCase						:       Dict  			=     ConfigTester(self				,     config_class=lowerCamelCase__				,     d_embed=37  )






     def     __A  (		self  ):
       self.config_tester.run_common_tests()






     def     __A  (		self  ):
       self.model_tester.set_seed()
       _lowerCAmelCase						:       List[Any]  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_transfo_xl_model(*lowerCamelCase__  )






     def     __A  (		self  ):
       self.model_tester.set_seed()
       _lowerCAmelCase						:       Tuple  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_transfo_xl_lm_head(*lowerCamelCase__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Optional[Any]  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_transfo_xl_for_sequence_classification(*lowerCamelCase__  )






     def     __A  (		self  ):
       _lowerCAmelCase			,	_lowerCAmelCase						:       Any  			=     self.model_tester.prepare_config_and_inputs_for_common()
       _lowerCAmelCase						:       Union[str, Any]  			=     [TFTransfoXLForSequenceClassification]

       for model_class in self.all_model_classes:
         _lowerCAmelCase						:       Dict  			=     model_class(lowerCamelCase__  )
         assert isinstance(model.get_input_embeddings()				,     tf.keras.layers.Layer  )
         if model_class in list_other_models_with_output_ebd:
           _lowerCAmelCase						:       List[Any]  			=     model.get_output_embeddings()
           assert isinstance(lowerCamelCase__				,     tf.keras.layers.Layer  )
           _lowerCAmelCase						:       Tuple  			=     model.get_bias()
           assert name is None
         else:
           _lowerCAmelCase						:       int  			=     model.get_output_embeddings()
           assert x is None
           _lowerCAmelCase						:       Union[str, Any]  			=     model.get_bias()
           assert name is None






     def     __A  (		self  ):
       # TODO JP: Make TransfoXL XLA compliant
       pass






     @slow
     def     __A  (		self  ):
       for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _lowerCAmelCase						:       Optional[Any]  			=     TFTransfoXLModel.from_pretrained(lowerCamelCase__  )
         self.assertIsNotNone(lowerCamelCase__  )






     @unittest.skip(reason="""This model doesn\'t play well with fit() due to not returning a single loss."""  )
     def     __A  (		self  ):
       pass







@require_tf
class 						__A							( unittest.TestCase		):






     @unittest.skip("""Skip test until #12651 is resolved."""  )
     @slow
     def     __A  (		self  ):
       _lowerCAmelCase						:       Dict  			=     TFTransfoXLLMHeadModel.from_pretrained("""transfo-xl-wt103"""  )
       # fmt: off
       _lowerCAmelCase						:       int  			=     tf.convert_to_tensor([[33,1297,2,1,1009,4,1109,11739,4762,358,5,25,245,22,1706,17,20098,5,3215,21,37,1110,3,13,1041,4,24,603,490,2,71477,20098,104447,2,20961,1,2604,4,1,329,3,6224,831,16002,2,8,603,78967,29546,23,803,20,25,416,5,8,232,4,277,6,1855,4601,3,29546,54,8,3609,5,57211,49,4,1,277,18,8,1755,15691,3,341,25,416,693,42573,71,17,401,94,31,17919,2,29546,7873,18,1,435,23,11011,755,5,5167,3,7983,98,84,2,29546,3267,8,3609,4,1,4865,1075,2,6087,71,6,346,8,5854,3,29546,824,1400,1868,2,19,160,2,311,8,5496,2,20920,17,25,15097,3,24,24,0]]				,     dtype=tf.intaa  )  # noqa: E231
       # fmt: on
       #  In 1991 , the remains of Russian Tsar Nicholas II and his family
       #  ( except for Alexei and Maria ) are discovered .
       #  The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the
       #  remainder of the story . 1883 Western Siberia ,
       #  a young Grigori Rasputin is asked by his father and a group of men to perform magic .
       #  Rasputin has a vision and denounces one of the men as a horse thief . Although his
       #  father initially slaps him for making such an accusation , Rasputin watches as the
       #  man is chased outside and beaten . Twenty years later , Rasputin sees a vision of
       #  the Virgin Mary , prompting him to become a priest . Rasputin quickly becomes famous ,
       #  with people , even a bishop , begging for his blessing . <eod> </s> <eos>

       # fmt: off
       _lowerCAmelCase						:       Optional[Any]  			=     [33,1297,2,1,1009,4,1109,11739,4762,358,5,25,245,22,1706,17,20098,5,3215,21,37,1110,3,13,1041,4,24,603,490,2,71477,20098,104447,2,20961,1,2604,4,1,329,3,6224,831,16002,2,8,603,78967,29546,23,803,20,25,416,5,8,232,4,277,6,1855,4601,3,29546,54,8,3609,5,57211,49,4,1,277,18,8,1755,15691,3,341,25,416,693,42573,71,17,401,94,31,17919,2,29546,7873,18,1,435,23,11011,755,5,5167,3,7983,98,84,2,29546,3267,8,3609,4,1,4865,1075,2,6087,71,6,346,8,5854,3,29546,824,1400,1868,2,19,160,2,311,8,5496,2,20920,17,25,15097,3,24,24,0,33,1,1857,2,1,1009,4,1109,11739,4762,358,5,25,245,28,1110,3,13,1041,4,24,603,490,2,71477,20098,104447,2,20961,1,2604,4,1,329,3,0]  # noqa: E231
       # fmt: on
       #  In 1991, the remains of Russian Tsar Nicholas II and his family (
       #  except for Alexei and Maria ) are discovered. The voice of young son,
       #  Tsarevich Alexei Nikolaevich, narrates the remainder of the story.
       #  1883 Western Siberia, a young Grigori Rasputin is asked by his father
       #  and a group of men to perform magic. Rasputin has a vision and
       #  denounces one of the men as a horse thief. Although his father initially
       #  slaps him for making such an accusation, Rasputin watches as the man
       #  is chased outside and beaten. Twenty years later, Rasputin sees a vision
       #  of the Virgin Mary, prompting him to become a priest.
       #  Rasputin quickly becomes famous, with people, even a bishop, begging for
       #  his blessing. <unk> <unk> <eos> In the 1990s, the remains of Russian Tsar
       # Nicholas II and his family were discovered. The voice of <unk> young son,
       # Tsarevich Alexei Nikolaevich, narrates the remainder of the story.<eos>

       _lowerCAmelCase						:       Union[str, Any]  			=     model.generate(lowerCamelCase__				,     max_length=200				,     do_sample=lowerCamelCase__  )
       self.assertListEqual(output_ids[0].numpy().tolist()				,     lowerCamelCase__  )





"""simple docstring"""






import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__  :  Optional[int]						     =	logging.get_logger(__name__)
class 				snake_case       ( __UpperCAmelCase				):




					"""simple docstring"""

					snake_case__				    = ["input_ids", "attention_mask"]







					def __init__(				self   :	Optional[int]    ,lowerCamelCase__   :	int="</s>"    ,lowerCamelCase__   :	str="<unk>"    ,lowerCamelCase__   :	Union[str, Any]="<pad>"    ,lowerCamelCase__   :	int=125    ,lowerCamelCase__   :	str=None    ,**lowerCamelCase__   :	Union[str, Any]    ,):
									# Add extra_ids to the special token list
									if extra_ids > 0 and additional_special_tokens is None:
													UpperCAmelCase__             =							[f'''<extra_id_{i}>''' for i in range(lowerCamelCase__			)]
									elif extra_ids > 0 and additional_special_tokens is not None:
													# Check that we have the right number of extra_id special tokens
													UpperCAmelCase__             =							len(set(filter(lambda lowerCamelCase__			: bool('extra_id' in str(lowerCamelCase__			)			)    ,lowerCamelCase__			)			)			)
													if extra_tokens != extra_ids:
																	raise ValueError(
																	    f'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
																	    ' provided to ByT5Tokenizer. In this case the additional_special_tokens must include the'
																	    ' extra_ids tokens'			)

									UpperCAmelCase__             =							AddedToken(lowerCamelCase__    ,lstrip=lowerCamelCase__    ,rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__    ,lowerCamelCase__			) else pad_token
									UpperCAmelCase__             =							AddedToken(lowerCamelCase__    ,lstrip=lowerCamelCase__    ,rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__    ,lowerCamelCase__			) else eos_token
									UpperCAmelCase__             =							AddedToken(lowerCamelCase__    ,lstrip=lowerCamelCase__    ,rstrip=lowerCamelCase__			) if isinstance(lowerCamelCase__    ,lowerCamelCase__			) else unk_token

									super().__init__(
									    eos_token=lowerCamelCase__    ,unk_token=lowerCamelCase__    ,pad_token=lowerCamelCase__    ,extra_ids=lowerCamelCase__    ,additional_special_tokens=lowerCamelCase__    ,**lowerCamelCase__    ,)

									UpperCAmelCase__             =							extra_ids

									UpperCAmelCase__             =							2**8  # utf is 8 bits

									# define special tokens dict
									UpperCAmelCase__             =							{
									    self.pad_token: 0,
									    self.eos_token: 1,
									    self.unk_token: 2,
									}
									UpperCAmelCase__             =							len(self.special_tokens_encoder			)
									UpperCAmelCase__             =							len(lowerCamelCase__			)
									for i, token in enumerate(lowerCamelCase__			):
													UpperCAmelCase__             =							self.vocab_size + i - n
									UpperCAmelCase__             =							{v: k for k, v in self.special_tokens_encoder.items()}







					@property
					def 		__lowerCAmelCase						(				self   :	Union[str, Any]			):
									return self._utf_vocab_size + self._num_special_tokens + self._extra_ids







					def 		__lowerCAmelCase						(				self   :	Optional[Any]    ,lowerCamelCase__   :	List[int]    ,lowerCamelCase__   :	Optional[List[int]] = None    ,lowerCamelCase__   :	bool = False			):
									if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=lowerCamelCase__    ,token_ids_a=lowerCamelCase__    ,already_has_special_tokens=lowerCamelCase__			)

									# normal case: some special tokens
									if token_ids_a is None:
													return ([0] * len(lowerCamelCase__			)) + [1]
									return ([0] * len(lowerCamelCase__			)) + [1] + ([0] * len(lowerCamelCase__			)) + [1]







					def 		__lowerCAmelCase						(				self   :	str    ,lowerCamelCase__   :	List[int]			):
									if len(lowerCamelCase__			) > 0 and token_ids[-1] == self.eos_token_id:
													warnings.warn(
													    f'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
													    ' eos tokens being added.'			)
													return token_ids
									else:
													return token_ids + [self.eos_token_id]







					def 		__lowerCAmelCase						(				self   :	str    ,lowerCamelCase__   :	List[int]    ,lowerCamelCase__   :	Optional[List[int]] = None			):
									UpperCAmelCase__             =							[self.eos_token_id]

									if token_ids_a is None:
													return len(token_ids_a + eos			) * [0]
									return len(token_ids_a + eos + token_ids_a + eos			) * [0]







					def 		__lowerCAmelCase						(				self   :	Dict    ,lowerCamelCase__   :	List[int]    ,lowerCamelCase__   :	Optional[List[int]] = None			):
									UpperCAmelCase__             =							self._add_eos_if_not_present(lowerCamelCase__			)
									if token_ids_a is None:
													return token_ids_a
									else:
													UpperCAmelCase__             =							self._add_eos_if_not_present(lowerCamelCase__			)
													return token_ids_a + token_ids_a







					def 		__lowerCAmelCase						(				self   :	Optional[Any]    ,lowerCamelCase__   :	str			):
									UpperCAmelCase__             =							[chr(lowerCamelCase__			) for i in text.encode('utf-8'			)]
									return tokens







					def 		__lowerCAmelCase						(				self   :	List[Any]    ,lowerCamelCase__   :	str			):
									if token in self.special_tokens_encoder:
													UpperCAmelCase__             =							self.special_tokens_encoder[token]
									elif token in self.added_tokens_encoder:
													UpperCAmelCase__             =							self.added_tokens_encoder[token]
									elif len(lowerCamelCase__			) != 1:
													UpperCAmelCase__             =							self.unk_token_id
									else:
													UpperCAmelCase__             =							ord(lowerCamelCase__			) + self._num_special_tokens
									return token_id







					def 		__lowerCAmelCase						(				self   :	Union[str, Any]    ,lowerCamelCase__   :	Any			):
									if index in self.special_tokens_decoder:
													UpperCAmelCase__             =							self.special_tokens_decoder[index]
									else:
													UpperCAmelCase__             =							chr(index - self._num_special_tokens			)
									return token







					def 		__lowerCAmelCase						(				self   :	Optional[Any]    ,lowerCamelCase__   :	Union[str, Any]			):
									UpperCAmelCase__             =							b''
									for token in tokens:
													if token in self.special_tokens_decoder:
																	UpperCAmelCase__             =							self.special_tokens_decoder[token].encode('utf-8'			)
													elif token in self.added_tokens_decoder:
																	UpperCAmelCase__             =							self.special_tokens_decoder[token].encode('utf-8'			)
													elif token in self.special_tokens_encoder:
																	UpperCAmelCase__             =							token.encode('utf-8'			)
													elif token in self.added_tokens_encoder:
																	UpperCAmelCase__             =							token.encode('utf-8'			)
													else:
																	UpperCAmelCase__             =							bytes([ord(lowerCamelCase__			)]			)
													bstring += tok_string
									UpperCAmelCase__             =							bstring.decode('utf-8'    ,errors='ignore'			)
									return string







					def 		__lowerCAmelCase						(				self   :	str    ,lowerCamelCase__   :	str    ,lowerCamelCase__   :	Optional[str] = None			):
									return ()



