


from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TFRoFormerForCausalLM,
				    TFRoFormerForMaskedLM,
				    TFRoFormerForMultipleChoice,
				    TFRoFormerForQuestionAnswering,
				    TFRoFormerForSequenceClassification,
				    TFRoFormerForTokenClassification,
				    TFRoFormerModel,
				)
				from transformers.models.roformer.modeling_tf_roformer import (
				    TFRoFormerSelfAttention,
				    TFRoFormerSinusoidalPositionalEmbedding,
				)






class    _UpperCamelCase     :

				"""simple docstring"""







				def __init__(    self					, a__					, a__=13					, a__=7					, a__=True					, a__=True					, a__=True					, a__=True					, a__=99					, a__=32					, a__=2					, a__=4					, a__=37					, a__="gelu"					, a__=0.1					, a__=0.1					, a__=512					, a__=16					, a__=2					, a__=0.02					, a__=3					, a__=4					, a__=None					, )						->					Dict:
					A             =					parent
					A             =					13
					A             =					7
					A             =					True
					A             =					True
					A             =					True
					A             =					True
					A             =					99
					A             =					32
					A             =					2
					A             =					4
					A             =					37
					A             =					"""gelu"""
					A             =					0.1
					A             =					0.1
					A             =					512
					A             =					16
					A             =					2
					A             =					0.02
					A             =					3
					A             =					4
					A             =					None

				def   _UpperCAmelCase       (    self )						->					Tuple:
					A             =					ids_tensor([self.batch_size, self.seq_length]					, self.vocab_size )

					A             =					None
					if self.use_input_mask:
						A             =					random_attention_mask([self.batch_size, self.seq_length] )

					A             =					None
					if self.use_token_type_ids:
						A             =					ids_tensor([self.batch_size, self.seq_length]					, self.type_vocab_size )

					A             =					None
					A             =					None
					A             =					None
					if self.use_labels:
						A             =					ids_tensor([self.batch_size]					, self.type_sequence_label_size )
						A             =					ids_tensor([self.batch_size, self.seq_length]					, self.num_labels )
						A             =					ids_tensor([self.batch_size]					, self.num_choices )

					A             =					RoFormerConfig(
					    vocab_size=self.vocab_size					, hidden_size=self.hidden_size					, num_hidden_layers=self.num_hidden_layers					, num_attention_heads=self.num_attention_heads					, intermediate_size=self.intermediate_size					, hidden_act=self.hidden_act					, hidden_dropout_prob=self.hidden_dropout_prob					, attention_probs_dropout_prob=self.attention_probs_dropout_prob					, max_position_embeddings=self.max_position_embeddings					, type_vocab_size=self.type_vocab_size					, initializer_range=self.initializer_range					, return_dict=a__					, )

					return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__ )						->					Optional[Any]:
					A             =					TFRoFormerModel(config=a__ )
					A             =					{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

					A             =					[input_ids, input_mask]
					A             =					model(a__ )

					A             =					model(a__ )

					self.parent.assertEqual(result.last_hidden_state.shape					, (self.batch_size, self.seq_length, self.hidden_size) )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__ )						->					List[Any]:
					A             =					True
					A             =					TFRoFormerForCausalLM(config=a__ )
					A             =					{
					    """input_ids""": input_ids,
					    """attention_mask""": input_mask,
					    """token_type_ids""": token_type_ids,
					}
					A             =					model(a__ )["""logits"""]
					self.parent.assertListEqual(
					    list(prediction_scores.numpy().shape )					, [self.batch_size, self.seq_length, self.vocab_size] )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__ )						->					Union[str, Any]:
					A             =					TFRoFormerForMaskedLM(config=a__ )
					A             =					{
					    """input_ids""": input_ids,
					    """attention_mask""": input_mask,
					    """token_type_ids""": token_type_ids,
					}
					A             =					model(a__ )
					self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.seq_length, self.vocab_size) )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__ )						->					Dict:
					A             =					self.num_labels
					A             =					TFRoFormerForSequenceClassification(config=a__ )
					A             =					{
					    """input_ids""": input_ids,
					    """attention_mask""": input_mask,
					    """token_type_ids""": token_type_ids,
					}

					A             =					model(a__ )
					self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.num_labels) )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__ )						->					Any:
					A             =					self.num_choices
					A             =					TFRoFormerForMultipleChoice(config=a__ )
					A             =					tf.tile(tf.expand_dims(a__					, 1 )					, (1, self.num_choices, 1) )
					A             =					tf.tile(tf.expand_dims(a__					, 1 )					, (1, self.num_choices, 1) )
					A             =					tf.tile(tf.expand_dims(a__					, 1 )					, (1, self.num_choices, 1) )
					A             =					{
					    """input_ids""": multiple_choice_inputs_ids,
					    """attention_mask""": multiple_choice_input_mask,
					    """token_type_ids""": multiple_choice_token_type_ids,
					}
					A             =					model(a__ )
					self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.num_choices) )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__ )						->					str:
					A             =					self.num_labels
					A             =					TFRoFormerForTokenClassification(config=a__ )
					A             =					{
					    """input_ids""": input_ids,
					    """attention_mask""": input_mask,
					    """token_type_ids""": token_type_ids,
					}
					A             =					model(a__ )
					self.parent.assertEqual(result.logits.shape					, (self.batch_size, self.seq_length, self.num_labels) )

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__					, a__					, a__ )						->					Optional[int]:
					A             =					TFRoFormerForQuestionAnswering(config=a__ )
					A             =					{
					    """input_ids""": input_ids,
					    """attention_mask""": input_mask,
					    """token_type_ids""": token_type_ids,
					}

					A             =					model(a__ )
					self.parent.assertEqual(result.start_logits.shape					, (self.batch_size, self.seq_length) )
					self.parent.assertEqual(result.end_logits.shape					, (self.batch_size, self.seq_length) )

				def   _UpperCAmelCase       (    self )						->					int:
					A             =					self.prepare_config_and_inputs()
					(
					    (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      (
					    A
					)       ,      
					)             =					config_and_inputs
					A             =					{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
					return config, inputs_dict






@require_tf
class    _UpperCamelCase     (       __snake_case     ,		__snake_case     ,		unittest.TestCase					):

				"""simple docstring"""







				lowerCAmelCase					=			(
				    (
				        TFRoFormerModel,
				        TFRoFormerForCausalLM,
				        TFRoFormerForMaskedLM,
				        TFRoFormerForQuestionAnswering,
				        TFRoFormerForSequenceClassification,
				        TFRoFormerForTokenClassification,
				        TFRoFormerForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				lowerCAmelCase					=			(
				    {
				        'feature-extraction': TFRoFormerModel,
				        'fill-mask': TFRoFormerForMaskedLM,
				        'question-answering': TFRoFormerForQuestionAnswering,
				        'text-classification': TFRoFormerForSequenceClassification,
				        'text-generation': TFRoFormerForCausalLM,
				        'token-classification': TFRoFormerForTokenClassification,
				        'zero-shot': TFRoFormerForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)

				lowerCAmelCase					=			False
				lowerCAmelCase					=			False

				def   _UpperCAmelCase       (    self					, a__					, a__					, a__					, a__					, a__ )						->					Optional[Any]:
					if pipeline_test_casse_name == "TextGenerationPipelineTests":
						return True

					return False

				def   _UpperCAmelCase       (    self )						->					Any:
					A             =					TFRoFormerModelTester(self )
					A             =					ConfigTester(self					, config_class=a__					, hidden_size=37 )

				def   _UpperCAmelCase       (    self )						->					str:
					self.config_tester.run_common_tests()

				def   _UpperCAmelCase       (    self )						->					Dict:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(*a__ )

				def   _UpperCAmelCase       (    self )						->					str:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_masked_lm(*a__ )

				def   _UpperCAmelCase       (    self )						->					List[Any]:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_lm_head(*a__ )

				def   _UpperCAmelCase       (    self )						->					List[str]:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_multiple_choice(*a__ )

				def   _UpperCAmelCase       (    self )						->					Dict:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_question_answering(*a__ )

				def   _UpperCAmelCase       (    self )						->					Any:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_sequence_classification(*a__ )

				def   _UpperCAmelCase       (    self )						->					Union[str, Any]:
					A             =					self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_token_classification(*a__ )

				@slow
				def   _UpperCAmelCase       (    self )						->					str:
					A             =					TFRoFormerModel.from_pretrained("""junnyu/roformer_chinese_base""" )
					self.assertIsNotNone(a__ )






@require_tf
class    _UpperCamelCase     (       unittest.TestCase					):

				"""simple docstring"""







				@slow
				def   _UpperCAmelCase       (    self )						->					List[Any]:
					A             =					TFRoFormerForMaskedLM.from_pretrained("""junnyu/roformer_chinese_base""" )
					A             =					tf.constant([[0, 1, 2, 3, 4, 5]] )
					A             =					model(a__ )[0]

					# TODO Replace vocab size
					A             =					5_0000

					A             =					[1, 6, vocab_size]
					self.assertEqual(output.shape					, a__ )

					print(output[:, :3, :3] )

					# TODO Replace values below with what was printed above.
					A             =					tf.constant(
					    [
					        [
					            [-0.12_05_33_41, -1.0_26_49_01, 0.29_22_19_46],
					            [-1.5_13_37_83, 0.19_74_33, 0.15_19_06_07],
					            [-5.0_13_54_03, -3.90_02_56, -0.84_03_87_64],
					        ]
					    ] )
					tf.debugging.assert_near(output[:, :3, :3]					, a__					, atol=1e-4 )






@require_tf
class    _UpperCamelCase     (       unittest.TestCase					):

				"""simple docstring"""







				lowerCAmelCase					=			1E-4

				def   _UpperCAmelCase       (    self )						->					Union[str, Any]:
					A             =					tf.constant([[4, 10]] )
					A             =					TFRoFormerSinusoidalPositionalEmbedding(num_positions=6					, embedding_dim=6 )

					A             =					emba(input_ids.shape )
					A             =					tf.constant(
					    [[0.00_00, 0.00_00, 0.00_00, 1.00_00, 1.00_00, 1.00_00], [0.84_15, 0.04_64, 0.00_22, 0.54_03, 0.99_89, 1.00_00]] )

					tf.debugging.assert_near(a__					, a__					, atol=self.tolerance )

				def   _UpperCAmelCase       (    self )						->					int:
					A             =					tf.constant(
					    [
					        [0.00_00, 0.00_00, 0.00_00, 0.00_00, 0.00_00],
					        [0.84_15, 0.82_19, 0.80_20, 0.78_19, 0.76_17],
					        [0.90_93, 0.93_64, 0.95_81, 0.97_49, 0.98_70],
					    ] )
					A             =					TFRoFormerSinusoidalPositionalEmbedding(num_positions=512					, embedding_dim=512 )
					emba([2, 16, 512] )
					A             =					emba.weight[:3, :5]

					tf.debugging.assert_near(a__					, a__					, atol=self.tolerance )






@require_tf
class    _UpperCamelCase     (       unittest.TestCase					):

				"""simple docstring"""







				lowerCAmelCase					=			1E-4

				def   _UpperCAmelCase       (    self )						->					str:
					# 2,12,16,64
					A             =					tf.reshape(tf.range(2 * 12 * 16 * 64					, dtype=tf.floataa )					, shape=(2, 12, 16, 64) ) / 100

					A             =					-tf.reshape(tf.range(2 * 12 * 16 * 64					, dtype=tf.floataa )					, shape=(2, 12, 16, 64) ) / 100

					A             =					TFRoFormerSinusoidalPositionalEmbedding(num_positions=32					, embedding_dim=64 )
					A             =					embed_positions([2, 16, 768] )[None, None, :, :]

					A       ,      A             =					TFRoFormerSelfAttention.apply_rotary_position_embeddings(
					    a__					, a__					, a__ )

					A             =					tf.constant(
					    [
					        [0.00_00, 0.01_00, 0.02_00, 0.03_00, 0.04_00, 0.05_00, 0.06_00, 0.07_00],
					        [-0.20_12, 0.88_97, 0.02_63, 0.94_01, 0.20_74, 0.94_63, 0.34_81, 0.93_43],
					        [-1.70_57, 0.62_71, -1.21_45, 1.38_97, -0.63_03, 1.76_47, -0.11_73, 1.89_85],
					        [-2.17_31, -1.63_97, -2.73_58, 0.28_54, -2.18_40, 1.71_83, -1.30_18, 2.48_71],
					        [0.27_17, -3.61_73, -2.92_06, -2.19_88, -3.66_38, 0.38_58, -2.91_55, 2.29_80],
					        [3.98_59, -2.15_80, -0.79_84, -4.49_04, -4.11_81, -2.02_52, -4.47_82, 1.12_53],
					    ] )
					A             =					tf.constant(
					    [
					        [0.00_00, -0.01_00, -0.02_00, -0.03_00, -0.04_00, -0.05_00, -0.06_00, -0.07_00],
					        [0.20_12, -0.88_97, -0.02_63, -0.94_01, -0.20_74, -0.94_63, -0.34_81, -0.93_43],
					        [1.70_57, -0.62_71, 1.21_45, -1.38_97, 0.63_03, -1.76_47, 0.11_73, -1.89_85],
					        [2.17_31, 1.63_97, 2.73_58, -0.28_54, 2.18_40, -1.71_83, 1.30_18, -2.48_71],
					        [-0.27_17, 3.61_73, 2.92_06, 2.19_88, 3.66_38, -0.38_58, 2.91_55, -2.29_80],
					        [-3.98_59, 2.15_80, 0.79_84, 4.49_04, 4.11_81, 2.02_52, 4.47_82, -1.12_53],
					    ] )

					tf.debugging.assert_near(query_layer[0, 0, :6, :8]					, a__					, atol=self.tolerance )
					tf.debugging.assert_near(key_layer[0, 0, :6, :8]					, a__					, atol=self.tolerance )







_lowercase							:   Dict						=						{
    "Pillow": "Pillow",
    "accelerate": "accelerate>=0.11.0",
    "compel": "compel==0.1.8",
    "black": "black~=23.1",
    "datasets": "datasets",
    "filelock": "filelock",
    "flax": "flax>=0.4.1",
    "hf-doc-builder": "hf-doc-builder>=0.3.0",
    "huggingface-hub": "huggingface-hub>=0.13.2",
    "requests-mock": "requests-mock==1.10.0",
    "importlib_metadata": "importlib_metadata",
    "invisible-watermark": "invisible-watermark",
    "isort": "isort>=5.5.4",
    "jax": "jax>=0.2.8,!=0.3.2",
    "jaxlib": "jaxlib>=0.1.65",
    "Jinja2": "Jinja2",
    "k-diffusion": "k-diffusion>=0.0.12",
    "torchsde": "torchsde",
    "note_seq": "note_seq",
    "librosa": "librosa",
    "numpy": "numpy",
    "omegaconf": "omegaconf",
    "parameterized": "parameterized",
    "protobuf": "protobuf>=3.20.3,<4",
    "pytest": "pytest",
    "pytest-timeout": "pytest-timeout",
    "pytest-xdist": "pytest-xdist",
    "ruff": "ruff>=0.0.241",
    "safetensors": "safetensors",
    "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92",
    "scipy": "scipy",
    "onnx": "onnx",
    "regex": "regex!=2019.12.17",
    "requests": "requests",
    "tensorboard": "tensorboard",
    "torch": "torch>=1.4",
    "torchvision": "torchvision",
    "transformers": "transformers>=4.25.1",
    "urllib3": "urllib3<=2.0.0",
}



