


'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import BlenderbotConfig, BlenderbotTokenizer, is_tf_available
from transformers.testing_utils import require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import TFAutoModelForSeqaSeqLM, TFBlenderbotForConditionalGeneration, TFBlenderbotModel
@require_tf
class   __lowerCAmelCase						:





      '''simple docstring'''






      lowerCAmelCase__					:     Union[str, Any]  					=							BlenderbotConfig
      lowerCAmelCase__					:     Dict  					=							{}
      lowerCAmelCase__					:     List[Any]  					=							"""gelu"""



      def __init__(self    :    str   ,						UpperCamelCase    :    Dict   ,						UpperCamelCase    :    Any=13   ,						UpperCamelCase    :    List[Any]=7   ,						UpperCamelCase    :    str=True   ,						UpperCamelCase    :    Dict=False   ,						UpperCamelCase    :    Optional[Any]=99   ,						UpperCamelCase    :    Dict=32   ,						UpperCamelCase    :    int=2   ,						UpperCamelCase    :    str=4   ,						UpperCamelCase    :    Any=37   ,						UpperCamelCase    :    str=0.1   ,						UpperCamelCase    :    int=0.1   ,						UpperCamelCase    :    Dict=20   ,						UpperCamelCase    :    Union[str, Any]=2   ,						UpperCamelCase    :    Optional[int]=1   ,						UpperCamelCase    :    Optional[int]=0   ,						):

             '''simple docstring'''







             lowercase__      			= parent
             lowercase__      			= batch_size
             lowercase__      			= seq_length
             lowercase__      			= is_training
             lowercase__      			= use_labels
             lowercase__      			= vocab_size
             lowercase__      			= hidden_size
             lowercase__      			= num_hidden_layers
             lowercase__      			= num_attention_heads
             lowercase__      			= intermediate_size

             lowercase__      			= hidden_dropout_prob
             lowercase__      			= attention_probs_dropout_prob
             lowercase__      			= max_position_embeddings
             lowercase__      			= eos_token_id
             lowercase__      			= pad_token_id
             lowercase__      			= bos_token_id



      def       UpperCamelCase__		(self    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= ids_tensor([self.batch_size, self.seq_length - 1]   ,						self.vocab_size )
             lowercase__      			= tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size )   ,						1 )
             lowercase__      			= tf.concat([input_ids, eos_tensor]   ,						axis=1 )

             lowercase__      			= ids_tensor([self.batch_size, self.seq_length]   ,						self.vocab_size )

             lowercase__      			= self.config_cls(
                 vocab_size=self.vocab_size   ,						d_model=self.hidden_size   ,						encoder_layers=self.num_hidden_layers   ,						decoder_layers=self.num_hidden_layers   ,						encoder_attention_heads=self.num_attention_heads   ,						decoder_attention_heads=self.num_attention_heads   ,						encoder_ffn_dim=self.intermediate_size   ,						decoder_ffn_dim=self.intermediate_size   ,						dropout=self.hidden_dropout_prob   ,						attention_dropout=self.attention_probs_dropout_prob   ,						max_position_embeddings=self.max_position_embeddings   ,						eos_token_ids=[2]   ,						bos_token_id=self.bos_token_id   ,						pad_token_id=self.pad_token_id   ,						decoder_start_token_id=self.pad_token_id   ,						**self.config_updates   ,						)
             lowercase__      			= prepare_blenderbot_inputs_dict(UpperCamelCase   ,						UpperCamelCase   ,						UpperCamelCase )
             return config, inputs_dict






      def       UpperCamelCase__		(self    :    List[Any]   ,						UpperCamelCase    :    List[str]   ,						UpperCamelCase    :    Dict ):

             '''simple docstring'''







             lowercase__      			= TFBlenderbotModel(config=UpperCamelCase ).get_decoder()
             lowercase__      			= inputs_dict['''input_ids''']

             lowercase__      			= input_ids[:1, :]
             lowercase__      			= inputs_dict['''attention_mask'''][:1, :]
             lowercase__      			= inputs_dict['''head_mask''']
             lowercase__      			= 1

             # first forward pass
             lowercase__      			= model(UpperCamelCase   ,						attention_mask=UpperCamelCase   ,						head_mask=UpperCamelCase   ,						use_cache=UpperCamelCase )

             lowercase__			,lowercase__      			= outputs.to_tuple()

             # create hypothetical next token and extent to next_input_ids
             lowercase__      			= ids_tensor((self.batch_size, 3)   ,						config.vocab_size )
             lowercase__      			= tf.cast(ids_tensor((self.batch_size, 3)   ,						2 )   ,						tf.inta )

             # append to next input_ids and
             lowercase__      			= tf.concat([input_ids, next_tokens]   ,						axis=-1 )
             lowercase__      			= tf.concat([attention_mask, next_attn_mask]   ,						axis=-1 )

             lowercase__      			= model(UpperCamelCase   ,						attention_mask=UpperCamelCase )[0]
             lowercase__      			= model(UpperCamelCase   ,						attention_mask=UpperCamelCase   ,						past_key_values=UpperCamelCase )[0]

             self.parent.assertEqual(next_tokens.shape[1]   ,						output_from_past.shape[1] )

             # select random slice
             lowercase__      			= int(ids_tensor((1,)   ,						output_from_past.shape[-1] ) )
             lowercase__      			= output_from_no_past[:, -3:, random_slice_idx]
             lowercase__      			= output_from_past[:, :, random_slice_idx]

             # test that outputs are equal for slice
             tf.debugging.assert_near(UpperCamelCase   ,						UpperCamelCase   ,						rtol=1E-3 )






def _SCREAMING_SNAKE_CASE      (A  , A  , A  , A=None  , A=None  , A=None  , A=None  , A=None  , )       ->       Any:






       """simple docstring"""







       if attention_mask is None:
              lowercase__      			= tf.cast(tf.math.not_equal(A  , config.pad_token_id       )  , tf.inta       )
       if decoder_attention_mask is None:
              lowercase__      			= tf.concat(
                  [
                      tf.ones(decoder_input_ids[:, :1].shape  , dtype=tf.inta       ),
                      tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]  , config.pad_token_id       )  , tf.inta       ),
                  ]  , axis=-1  , )
       if head_mask is None:
              lowercase__      			= tf.ones((config.encoder_layers, config.encoder_attention_heads)       )
       if decoder_head_mask is None:
              lowercase__      			= tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
       if cross_attn_head_mask is None:
              lowercase__      			= tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
       return {
           "input_ids": input_ids,
           "decoder_input_ids": decoder_input_ids,
           "attention_mask": attention_mask,
           "decoder_attention_mask": decoder_attention_mask,
           "head_mask": head_mask,
           "decoder_head_mask": decoder_head_mask,
           "cross_attn_head_mask": cross_attn_head_mask,
       }
@require_tf
class   __lowerCAmelCase						(lowercase_  ,     lowercase_  ,     unittest.TestCase     ):





      '''simple docstring'''






      lowerCAmelCase__					:     str  					=							(TFBlenderbotForConditionalGeneration, TFBlenderbotModel) if is_tf_available() else ()
      lowerCAmelCase__					:     Tuple  					=							(TFBlenderbotForConditionalGeneration,) if is_tf_available() else ()
      lowerCAmelCase__					:     List[Any]  					=							(
          {
              """conversational""": TFBlenderbotForConditionalGeneration,
              """feature-extraction""": TFBlenderbotModel,
              """summarization""": TFBlenderbotForConditionalGeneration,
              """text2text-generation""": TFBlenderbotForConditionalGeneration,
              """translation""": TFBlenderbotForConditionalGeneration,
          }
          if is_tf_available()
          else {}
      )
      lowerCAmelCase__					:     Optional[Any]  					=							True
      lowerCAmelCase__					:     Optional[Any]  					=							False
      lowerCAmelCase__					:     str  					=							False



      def       UpperCamelCase__		(self    :    Tuple ):

             '''simple docstring'''







             lowercase__      			= TFBlenderbotModelTester(self )
             lowercase__      			= ConfigTester(self   ,						config_class=UpperCamelCase )



      def       UpperCamelCase__		(self    :    Union[str, Any] ):

             '''simple docstring'''







             self.config_tester.run_common_tests()






      def       UpperCamelCase__		(self    :    List[Any] ):

             '''simple docstring'''







             lowercase__      			= self.model_tester.prepare_config_and_inputs_for_common()
             self.model_tester.check_decoder_model_past_large_inputs(*UpperCamelCase )






@require_tokenizers
@require_tf
class   __lowerCAmelCase						(unittest.TestCase     ):





      '''simple docstring'''






      lowerCAmelCase__					:     int  					=							["""My friends are cool but they eat too many carbs."""]
      lowerCAmelCase__					:     Any  					=							"""facebook/blenderbot-400M-distill"""



      @cached_property
      def       UpperCamelCase__		(self    :    List[str] ):

             '''simple docstring'''







             return BlenderbotTokenizer.from_pretrained(self.model_name )



      @cached_property
      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name )
             return model






      @slow
      def       UpperCamelCase__		(self    :    str ):

             '''simple docstring'''







             lowercase__      			= self.tokenizer(self.src_text   ,						return_tensors='''tf''' )
             lowercase__      			= self.model.generate(
                 model_inputs.input_ids   ,						)
             lowercase__      			= self.tokenizer.batch_decode(generated_ids.numpy()   ,						skip_special_tokens=UpperCamelCase )[0]
             assert (
                 generated_words
                 == " That's unfortunate. Are they trying to lose weight or are they just trying to be healthier?"
             )








"""simple docstring"""







import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase__      				=   '''▁'''

lowerCAmelCase__      				=   get_tests_dir('''fixtures/test_sentencepiece.model''')


@require_sentencepiece
@require_tokenizers
class       SCREAMING_SNAKE_CASE__				(		lowercase   ,    unittest.TestCase				):
       """simple docstring"""


       a  :				str   =BigBirdTokenizer
       a  :				Union[str, Any]   =BigBirdTokenizerFast
       a  :				Tuple   =True
       a  :				Any   =True

       def        lowercase__							(						self						):






         """simple docstring"""





         super().setUp()

         lowerCAmelCase     :     str								=							self.tokenizer_class(snake_case__		,     keep_accents=snake_case__						)
         tokenizer.save_pretrained(self.tmpdirname						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     str								=							"<s>"
         lowerCAmelCase     :     Optional[int]								=							1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__						)		,     snake_case__						)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__						)		,     snake_case__						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Dict								=							list(self.get_tokenizer().get_vocab().keys()						)

         self.assertEqual(vocab_keys[0]		,     "<unk>"						)
         self.assertEqual(vocab_keys[1]		,     "<s>"						)
         self.assertEqual(vocab_keys[-1]		,     "[MASK]"						)
         self.assertEqual(len(snake_case__						)		,     1_004						)

       def        lowercase__							(						self						):






         """simple docstring"""





         self.assertEqual(self.get_tokenizer().vocab_size		,     1_000						)

       def        lowercase__							(						self						):






         """simple docstring"""





         if not self.test_rust_tokenizer:
           return

         lowerCAmelCase     :     Tuple								=							self.get_tokenizer()
         lowerCAmelCase     :     Optional[int]								=							self.get_rust_tokenizer()

         lowerCAmelCase     :     Tuple								=							"I was born in 92000, and this is falsé."

         lowerCAmelCase     :     Optional[int]								=							tokenizer.tokenize(snake_case__						)
         lowerCAmelCase     :     int								=							rust_tokenizer.tokenize(snake_case__						)
         self.assertListEqual(snake_case__		,     snake_case__						)

         lowerCAmelCase     :     Tuple								=							tokenizer.encode(snake_case__		,     add_special_tokens=snake_case__						)
         lowerCAmelCase     :     int								=							rust_tokenizer.encode(snake_case__		,     add_special_tokens=snake_case__						)
         self.assertListEqual(snake_case__		,     snake_case__						)

         lowerCAmelCase     :     List[Any]								=							self.get_rust_tokenizer()
         lowerCAmelCase     :     Tuple								=							tokenizer.encode(snake_case__						)
         lowerCAmelCase     :     List[Any]								=							rust_tokenizer.encode(snake_case__						)
         self.assertListEqual(snake_case__		,     snake_case__						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Any								=							BigBirdTokenizer(snake_case__		,     keep_accents=snake_case__						)

         lowerCAmelCase     :     Union[str, Any]								=							tokenizer.tokenize("This is a test"						)
         self.assertListEqual(snake_case__		,     ["▁This", "▁is", "▁a", "▁t", "est"]						)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(snake_case__						)		,     [285, 46, 10, 170, 382]		,     )

         lowerCAmelCase     :     Any								=							tokenizer.tokenize("I was born in 92000, and this is falsé."						)
         self.assertListEqual(
             snake_case__		,     [
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "é",
                 ".",
             ]		,     )
         lowerCAmelCase     :     str								=							tokenizer.convert_tokens_to_ids(snake_case__						)
         self.assertListEqual(
             snake_case__		,     [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]		,     )

         lowerCAmelCase     :     List[str]								=							tokenizer.convert_ids_to_tokens(snake_case__						)
         self.assertListEqual(
             snake_case__		,     [
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "<unk>",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "<unk>",
                 ".",
             ]		,     )

       @cached_property
       def        lowercase__							(						self						):






         """simple docstring"""





         return BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base"						)

       @slow
       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     List[Any]								=							"Hello World!"
         lowerCAmelCase     :     Any								=							[65, 18_536, 2_260, 101, 66]

         self.assertListEqual(snake_case__		,     self.big_tokenizer.encode(snake_case__						)						)

       @slow
       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Union[str, Any]								=							(
             "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
             " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
         )
         # fmt: off
         lowerCAmelCase     :     List[str]								=							[65, 871, 419, 358, 946, 991, 2_521, 452, 358, 1_357, 387, 7_751, 3_536, 112, 985, 456, 126, 865, 938, 5_400, 5_734, 458, 1_368, 467, 786, 2_462, 5_246, 1_159, 633, 865, 4_519, 457, 582, 852, 2_557, 427, 916, 508, 405, 34_324, 497, 391, 408, 11_342, 1_244, 385, 100, 938, 985, 456, 574, 362, 12_597, 3_200, 3_129, 1_172, 66]  # noqa: E231
         # fmt: on
         self.assertListEqual(snake_case__		,     self.big_tokenizer.encode(snake_case__						)						)

       @require_torch
       @slow
       def        lowercase__							(						self						):






         """simple docstring"""





         import torch

         from transformers import BigBirdConfig, BigBirdModel

         # Build sequence
         lowerCAmelCase     :     Dict								=							list(self.big_tokenizer.get_vocab().keys()						)[:10]
         lowerCAmelCase     :     int								=							" ".join(snake_case__						)
         lowerCAmelCase     :     Dict								=							self.big_tokenizer.encode_plus(snake_case__		,     return_tensors="pt"		,     return_token_type_ids=snake_case__						)
         lowerCAmelCase     :     Any								=							self.big_tokenizer.batch_encode_plus(
             [sequence + " " + sequence]		,     return_tensors="pt"		,     return_token_type_ids=snake_case__						)

         lowerCAmelCase     :     str								=							BigBirdConfig(attention_type="original_full"						)
         lowerCAmelCase     :     Any								=							BigBirdModel(snake_case__						)

         assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

         with torch.no_grad():
           model(**snake_case__						)
           model(**snake_case__						)

       @slow
       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     List[str]								=							BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base"						)
         lowerCAmelCase     :     Union[str, Any]								=							tokenizer.decode(tokenizer("Paris is the [MASK]."						).input_ids						)

         self.assertTrue(decoded_text == "[CLS] Paris is the[MASK].[SEP]"						)







       @slow
       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Any								=							{"input_ids": [[65, 39_286, 458, 36_335, 2_001, 456, 13_073, 13_266, 455, 113, 7_746, 1_741, 11_157, 391, 13_073, 13_266, 455, 113, 3_967, 35_412, 113, 4_936, 109, 3_870, 2_377, 113, 30_084, 45_720, 458, 134, 17_496, 112, 503, 11_672, 113, 118, 112, 5_665, 13_347, 38_687, 112, 1_496, 31_389, 112, 3_268, 47_264, 134, 962, 112, 16_377, 8_035, 23_130, 430, 12_169, 15_518, 28_592, 458, 146, 41_697, 109, 391, 12_169, 15_518, 16_689, 458, 146, 41_358, 109, 452, 726, 4_034, 111, 763, 35_412, 5_082, 388, 1_903, 111, 9_051, 391, 2_870, 48_918, 1_900, 1_123, 550, 998, 112, 9_586, 15_985, 455, 391, 410, 22_955, 37_636, 114, 66], [65, 448, 17_496, 419, 3_663, 385, 763, 113, 27_533, 2_870, 3_283, 13_043, 1_639, 24_713, 523, 656, 24_013, 18_550, 2_521, 517, 27_014, 21_244, 420, 1_212, 1_465, 391, 927, 4_833, 388, 578, 11_786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2_169, 7_687, 21_932, 18_146, 726, 363, 17_032, 3_391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=snake_case__		,     model_name="google/bigbird-roberta-base"		,     revision="215c99f1600e06f83acce68422f2035b2b5c3510"		,     )





