





from __future__ import annotations

import unittest

from transformers import BlenderbotSmallConfig, BlenderbotSmallTokenizer, is_tf_available
from transformers.testing_utils import require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import TFAutoModelForSeqaSeqLM, TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel







@require_tf
class 				__lowerCAmelCase       :
      _lowercase				:					str						=			BlenderbotSmallConfig
      _lowercase				:					Optional[int]						=			{}
      _lowercase				:					int						=			"""gelu"""







      def __init__(    self		,      lowerCAmelCase__		,      lowerCAmelCase__=1_3		,      lowerCAmelCase__=7		,      lowerCAmelCase__=True		,      lowerCAmelCase__=False		,      lowerCAmelCase__=9_9		,      lowerCAmelCase__=3_2		,      lowerCAmelCase__=2		,      lowerCAmelCase__=4		,      lowerCAmelCase__=3_7		,      lowerCAmelCase__=0.1		,      lowerCAmelCase__=0.1		,      lowerCAmelCase__=2_0		,      lowerCAmelCase__=2		,      lowerCAmelCase__=1		,      lowerCAmelCase__=0		,      )   ->						str:



        '''simple docstring'''




        a__	:					List[str]										=parent
        a__	:					Union[str, Any]										=batch_size
        a__	:					Optional[Any]										=seq_length
        a__	:					str										=is_training
        a__	:					Any										=use_labels
        a__	:					Optional[Any]										=vocab_size
        a__	:					List[Any]										=hidden_size
        a__	:					List[str]										=num_hidden_layers
        a__	:					List[str]										=num_attention_heads
        a__	:					Tuple										=intermediate_size

        a__	:					Optional[int]										=hidden_dropout_prob
        a__	:					List[Any]										=attention_probs_dropout_prob
        a__	:					int										=max_position_embeddings
        a__	:					Optional[Any]										=eos_token_id
        a__	:					List[Any]										=pad_token_id
        a__	:					Any										=bos_token_id







      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        a__	:					int										=ids_tensor([self.batch_size, self.seq_length - 1]		,      self.vocab_size							)
        a__	:					int										=tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size							)		,      1							)
        a__	:					List[str]										=tf.concat([input_ids, eos_tensor]		,      axis=1							)

        a__	:					Any										=ids_tensor([self.batch_size, self.seq_length]		,      self.vocab_size							)

        a__	:					Dict										=self.config_cls(
            vocab_size=self.vocab_size		,      d_model=self.hidden_size		,      encoder_layers=self.num_hidden_layers		,      decoder_layers=self.num_hidden_layers		,      encoder_attention_heads=self.num_attention_heads		,      decoder_attention_heads=self.num_attention_heads		,      encoder_ffn_dim=self.intermediate_size		,      decoder_ffn_dim=self.intermediate_size		,      dropout=self.hidden_dropout_prob		,      attention_dropout=self.attention_probs_dropout_prob		,      max_position_embeddings=self.max_position_embeddings		,      eos_token_ids=[2]		,      bos_token_id=self.bos_token_id		,      pad_token_id=self.pad_token_id		,      decoder_start_token_id=self.pad_token_id		,      **self.config_updates		,      )
        a__	:					str										=prepare_blenderbot_small_inputs_dict(_snake_case		,      _snake_case		,      _snake_case							)
        return config, inputs_dict







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Tuple:



        '''simple docstring'''




        a__	:					str										=TFBlenderbotSmallModel(config=_snake_case							).get_decoder()
        a__	:					List[str]										=inputs_dict['input_ids']

        a__	:					Tuple										=input_ids[:1, :]
        a__	:					str										=inputs_dict['attention_mask'][:1, :]
        a__	:					Optional[Any]										=inputs_dict['head_mask']
        a__	:					Union[str, Any]										=1

        # first forward pass
        a__	:					Dict										=model(_snake_case		,      attention_mask=_snake_case		,      head_mask=_snake_case		,      use_cache=_snake_case							)

        a__	:					List[Any]										=outputs.to_tuple()

        # create hypothetical next token and extent to next_input_ids
        a__	:					Optional[int]										=ids_tensor((self.batch_size, 3)		,      config.vocab_size							)
        a__	:					Any										=tf.cast(ids_tensor((self.batch_size, 3)		,      2							)		,      tf.inta							)

        # append to next input_ids and
        a__	:					int										=tf.concat([input_ids, next_tokens]		,      axis=-1							)
        a__	:					List[str]										=tf.concat([attention_mask, next_attn_mask]		,      axis=-1							)

        a__	:					Tuple										=model(_snake_case		,      attention_mask=_snake_case							)[0]
        a__	:					int										=model(_snake_case		,      attention_mask=_snake_case		,      past_key_values=_snake_case							)[0]

        self.parent.assertEqual(next_tokens.shape[1]		,      output_from_past.shape[1]							)

        # select random slice
        a__	:					List[str]										=int(ids_tensor((1,)		,      output_from_past.shape[-1]							)							)
        a__	:					Tuple										=output_from_no_past[:, -3:, random_slice_idx]
        a__	:					List[Any]										=output_from_past[:, :, random_slice_idx]

        # test that outputs are equal for slice
        tf.debugging.assert_near(_snake_case		,      _snake_case		,      rtol=1E-3							)
def      _A       (		SCREAMING_SNAKE_CASE     :   Dict       ,       SCREAMING_SNAKE_CASE     :   Any       ,       SCREAMING_SNAKE_CASE     :   Optional[int]       ,       SCREAMING_SNAKE_CASE     :   Dict=None       ,       SCREAMING_SNAKE_CASE     :   int=None       ,       SCREAMING_SNAKE_CASE     :   List[str]=None       ,       SCREAMING_SNAKE_CASE     :   Union[str, Any]=None       ,       SCREAMING_SNAKE_CASE     :   Optional[Any]=None       ,       ):





  """simple docstring"""



  if attention_mask is None:
    a__	:					Union[str, Any]										=tf.cast(tf.math.not_equal(SCREAMING_SNAKE_CASE       ,       config.pad_token_id						)       ,       tf.inta						)
  if decoder_attention_mask is None:
    a__	:					List[str]										=tf.concat(
        [
            tf.ones(decoder_input_ids[:, :1].shape       ,       dtype=tf.inta						),
            tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]       ,       config.pad_token_id						)       ,       tf.inta						),
        ]       ,       axis=-1       ,       )
  if head_mask is None:
    a__	:					Tuple										=tf.ones((config.encoder_layers, config.encoder_attention_heads)						)
  if decoder_head_mask is None:
    a__	:					int										=tf.ones((config.decoder_layers, config.decoder_attention_heads)						)
  if cross_attn_head_mask is None:
    a__	:					Any										=tf.ones((config.decoder_layers, config.decoder_attention_heads)						)
  return {
      "input_ids": input_ids,
      "decoder_input_ids": decoder_input_ids,
      "attention_mask": attention_mask,
      "decoder_attention_mask": decoder_attention_mask,
      "head_mask": head_mask,
      "decoder_head_mask": decoder_head_mask,
      "cross_attn_head_mask": cross_attn_head_mask,
  }







@require_tf
class 				__lowerCAmelCase       (    lowerCAmelCase_						,				lowerCAmelCase_						,				unittest.TestCase):
      _lowercase				:					List[str]						=			(
          (TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel) if is_tf_available() else ()
      )
      _lowercase				:					Optional[Any]						=			(TFBlenderbotSmallForConditionalGeneration,) if is_tf_available() else ()
      _lowercase				:					Dict						=			(
          {
              """conversational""": TFBlenderbotSmallForConditionalGeneration,
              """feature-extraction""": TFBlenderbotSmallModel,
              """summarization""": TFBlenderbotSmallForConditionalGeneration,
              """text2text-generation""": TFBlenderbotSmallForConditionalGeneration,
              """translation""": TFBlenderbotSmallForConditionalGeneration,
          }
          if is_tf_available()
          else {}
      )
      _lowercase				:					Tuple						=			True
      _lowercase				:					List[str]						=			False
      _lowercase				:					List[Any]						=			False







      def    _lowercase  (    self							)   ->						List[str]:



        '''simple docstring'''




        a__	:					Optional[int]										=TFBlenderbotSmallModelTester(self							)
        a__	:					List[Any]										=ConfigTester(self		,      config_class=_snake_case							)







      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        self.config_tester.run_common_tests()







      def    _lowercase  (    self							)   ->						Optional[Any]:



        '''simple docstring'''




        a__	:					List[Any]										=self.model_tester.prepare_config_and_inputs_for_common()
        self.model_tester.check_decoder_model_past_large_inputs(*_snake_case							)







@require_tokenizers
@require_tf
class 				__lowerCAmelCase       (    unittest.TestCase):
      _lowercase				:					Dict						=			[
          """Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like """
          """  i'm going to throw up.\nand why is that?"""
      ]
      _lowercase				:					int						=			"""facebook/blenderbot_small-90M"""







      @cached_property
      def    _lowercase  (    self							)   ->						List[Any]:



        '''simple docstring'''




        return BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot-90M"							)







      @cached_property
      def    _lowercase  (    self							)   ->						List[str]:



        '''simple docstring'''




        a__	:					Tuple										=TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name							)
        return model







      @slow
      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        a__	:					Any										=self.tokenizer(self.src_text		,      return_tensors="tf"							)
        a__	:					List[str]										=self.model.generate(
            model_inputs.input_ids		,      attention_mask=model_inputs.attention_mask		,      num_beams=2		,      use_cache=_snake_case		,      )
        a__	:					List[str]										=self.tokenizer.batch_decode(generated_ids.numpy()		,      skip_special_tokens=_snake_case							)[0]
        assert generated_words in (
            "i don't know. i just feel like i'm going to throw up. it's not fun.",
            "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
            "i'm not sure. i just feel like i've been in a bad situation.",
        )











import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin




class 					snake_case__						(       lowerCAmelCase_				,     unittest.TestCase     ):






     """simple docstring"""


     _SCREAMING_SNAKE_CASE						=	TransfoXLTokenizer
     _SCREAMING_SNAKE_CASE						=	False
     _SCREAMING_SNAKE_CASE						=	False






     def    lowercase_   (       self  :  Optional[int]					)   ->Any:
            super().setUp()

            snake_case__							:	Tuple   	=	[
                '<unk>',
                '[CLS]',
                '[SEP]',
                'want',
                'unwanted',
                'wa',
                'un',
                'running',
                ',',
                'low',
                'l',
            ]
            snake_case__							:	Any   	=	os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file']					)
            with open(self.vocab_file, 'w', encoding='utf-8'					) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)






     def    lowercase_   (       self  :  Union[str, Any], **_snake_case  :  List[Any]					)   ->Dict:
            snake_case__							:	str   	=	True
            return TransfoXLTokenizer.from_pretrained(self.tmpdirname, **_snake_case					)






     def    lowercase_   (       self  :  Optional[Any], _snake_case  :  str					)   ->Dict:
            snake_case__							:	List[Any]   	=	'<unk> UNwanted , running'
            snake_case__							:	List[Any]   	=	'<unk> unwanted, running'
            return input_text, output_text






     def    lowercase_   (       self  :  List[Any]					)   ->Tuple:
            snake_case__							:	Dict   	=	TransfoXLTokenizer(vocab_file=self.vocab_file, lower_case=_snake_case					)

            snake_case__							:	str   	=	tokenizer.tokenize('<unk> UNwanted , running'					)
            self.assertListEqual(_snake_case, ['<unk>', 'unwanted', ',', 'running']					)

            self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case					), [0, 4, 8, 7]					)






     def    lowercase_   (       self  :  List[str]					)   ->List[Any]:
            snake_case__							:	str   	=	TransfoXLTokenizer(lower_case=_snake_case					)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '					), ['hello', '!', 'how', 'are', 'you', '?']					)






     def    lowercase_   (       self  :  Optional[int]					)   ->Optional[Any]:
            snake_case__							:	Optional[int]   	=	TransfoXLTokenizer(lower_case=_snake_case					)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '					), ['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)






     def    lowercase_   (       self  :  Optional[int]					)   ->Union[str, Any]:
            snake_case__							:	List[Any]   	=	TransfoXLTokenizer(lower_case=_snake_case					)
            snake_case__							:	Dict   	=	'Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
            snake_case__							:	List[Any]   	=	[
                'Hello',
                '(',
                'bracket',
                ')',
                'and',
                'side',
                '@-@',
                'scrolled',
                '[',
                'and',
                ']',
                'Henry',
                '\'s',
                '$',
                '5',
                '@,@',
                '000',
                'with',
                '3',
                '@.@',
                '34',
                'm',
                '.',
                'What',
                '\'s',
                'up',
                '!',
                '?',
            ]

            self.assertListEqual(tokenizer.tokenize(_snake_case					), _snake_case					)

            self.assertEqual(tokenizer.convert_tokens_to_string(_snake_case					), _snake_case					)






     def    lowercase_   (       self  :  Dict					)   ->Any:
            snake_case__							:	Dict   	=	self.get_tokenizer()
            snake_case__							:	Optional[Any]   	=	len(_snake_case					)

            tokenizer.add_tokens(['new1', 'new2']					)
            tokenizer.move_added_token('new1', 1					)

            # Check that moved token is not copied (duplicate)
            self.assertEqual(len(_snake_case					), original_len + 2					)
            # Check that token is moved to specified id
            self.assertEqual(tokenizer.encode('new1'					), [1]					)
            self.assertEqual(tokenizer.decode([1]					), 'new1'					)






