
'''simple docstring'''




from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import (
          TFRoFormerForCausalLM,
          TFRoFormerForMaskedLM,
          TFRoFormerForMultipleChoice,
          TFRoFormerForQuestionAnswering,
          TFRoFormerForSequenceClassification,
          TFRoFormerForTokenClassification,
          TFRoFormerModel,
      )
      from transformers.models.roformer.modeling_tf_roformer import (
          TFRoFormerSelfAttention,
          TFRoFormerSinusoidalPositionalEmbedding,
      )





class 					A							:





     def __init__(				self		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE=13		,     SCREAMING_SNAKE_CASE=7		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE=True		,     SCREAMING_SNAKE_CASE=99		,     SCREAMING_SNAKE_CASE=32		,     SCREAMING_SNAKE_CASE=2		,     SCREAMING_SNAKE_CASE=4		,     SCREAMING_SNAKE_CASE=37		,     SCREAMING_SNAKE_CASE="gelu"		,     SCREAMING_SNAKE_CASE=0.1		,     SCREAMING_SNAKE_CASE=0.1		,     SCREAMING_SNAKE_CASE=512		,     SCREAMING_SNAKE_CASE=16		,     SCREAMING_SNAKE_CASE=2		,     SCREAMING_SNAKE_CASE=0.02		,     SCREAMING_SNAKE_CASE=3		,     SCREAMING_SNAKE_CASE=4		,     SCREAMING_SNAKE_CASE=None		,     )					->							str:






         """simple docstring"""






         A			:  Optional[int]   =		parent
         A			:  Optional[int]   =		13
         A			:  Optional[int]   =		7
         A			:  Dict   =		True
         A			:  str   =		True
         A			:  Any   =		True
         A			:  Optional[Any]   =		True
         A			:  Tuple   =		99
         A			:  Tuple   =		32
         A			:  Optional[Any]   =		2
         A			:  Any   =		4
         A			:  Tuple   =		37
         A			:  Any   =		'''gelu'''
         A			:  Union[str, Any]   =		0.1
         A			:  List[str]   =		0.1
         A			:  Union[str, Any]   =		512
         A			:  Union[str, Any]   =		16
         A			:  Union[str, Any]   =		2
         A			:  str   =		0.02
         A			:  int   =		3
         A			:  Optional[int]   =		4
         A			:  str   =		None





     def 							__lowerCAmelCase				(				self      )					->							Tuple:






         """simple docstring"""






         A			:  List[str]   =		ids_tensor([self.batch_size, self.seq_length]		,     self.vocab_size      )

         A			:  int   =		None
         if self.use_input_mask:
             A			:  List[str]   =		random_attention_mask([self.batch_size, self.seq_length]      )

         A			:  str   =		None
         if self.use_token_type_ids:
             A			:  List[Any]   =		ids_tensor([self.batch_size, self.seq_length]		,     self.type_vocab_size      )

         A			:  str   =		None
         A			:  str   =		None
         A			:  int   =		None
         if self.use_labels:
             A			:  Dict   =		ids_tensor([self.batch_size]		,     self.type_sequence_label_size      )
             A			:  str   =		ids_tensor([self.batch_size, self.seq_length]		,     self.num_labels      )
             A			:  Optional[Any]   =		ids_tensor([self.batch_size]		,     self.num_choices      )

         A			:  Any   =		RoFormerConfig(
             vocab_size=self.vocab_size		,     hidden_size=self.hidden_size		,     num_hidden_layers=self.num_hidden_layers		,     num_attention_heads=self.num_attention_heads		,     intermediate_size=self.intermediate_size		,     hidden_act=self.hidden_act		,     hidden_dropout_prob=self.hidden_dropout_prob		,     attention_probs_dropout_prob=self.attention_probs_dropout_prob		,     max_position_embeddings=self.max_position_embeddings		,     type_vocab_size=self.type_vocab_size		,     initializer_range=self.initializer_range		,     return_dict=SCREAMING_SNAKE_CASE		,     )

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





     def 							__lowerCAmelCase				(				self		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )					->							Tuple:






         """simple docstring"""






         A			:  Optional[int]   =		TFRoFormerModel(config=SCREAMING_SNAKE_CASE      )
         A			:  List[str]   =		{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

         A			:  List[str]   =		[input_ids, input_mask]
         A			:  Any   =		model(SCREAMING_SNAKE_CASE      )

         A			:  Union[str, Any]   =		model(SCREAMING_SNAKE_CASE      )

         self.parent.assertEqual(result.last_hidden_state.shape		,     (self.batch_size, self.seq_length, self.hidden_size)      )





     def 							__lowerCAmelCase				(				self		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )					->							Tuple:






         """simple docstring"""






         A			:  int   =		True
         A			:  Dict   =		TFRoFormerForCausalLM(config=SCREAMING_SNAKE_CASE      )
         A			:  List[Any]   =		{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         A			:  Optional[Any]   =		model(SCREAMING_SNAKE_CASE      )['''logits''']
         self.parent.assertListEqual(
             list(prediction_scores.numpy().shape      )		,     [self.batch_size, self.seq_length, self.vocab_size]      )





     def 							__lowerCAmelCase				(				self		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )					->							Dict:






         """simple docstring"""






         A			:  Union[str, Any]   =		TFRoFormerForMaskedLM(config=SCREAMING_SNAKE_CASE      )
         A			:  Optional[Any]   =		{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         A			:  Optional[int]   =		model(SCREAMING_SNAKE_CASE      )
         self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.vocab_size)      )





     def 							__lowerCAmelCase				(				self		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )					->							Tuple:






         """simple docstring"""






         A			:  List[Any]   =		self.num_labels
         A			:  List[str]   =		TFRoFormerForSequenceClassification(config=SCREAMING_SNAKE_CASE      )
         A			:  Optional[int]   =		{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }

         A			:  Dict   =		model(SCREAMING_SNAKE_CASE      )
         self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_labels)      )





     def 							__lowerCAmelCase				(				self		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )					->							Union[str, Any]:






         """simple docstring"""






         A			:  Optional[int]   =		self.num_choices
         A			:  Tuple   =		TFRoFormerForMultipleChoice(config=SCREAMING_SNAKE_CASE      )
         A			:  List[Any]   =		tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE		,     1      )		,     (1, self.num_choices, 1)      )
         A			:  Optional[Any]   =		tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE		,     1      )		,     (1, self.num_choices, 1)      )
         A			:  Tuple   =		tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE		,     1      )		,     (1, self.num_choices, 1)      )
         A			:  Dict   =		{
             '''input_ids''': multiple_choice_inputs_ids,
             '''attention_mask''': multiple_choice_input_mask,
             '''token_type_ids''': multiple_choice_token_type_ids,
         }
         A			:  Any   =		model(SCREAMING_SNAKE_CASE      )
         self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_choices)      )





     def 							__lowerCAmelCase				(				self		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )					->							Any:






         """simple docstring"""






         A			:  int   =		self.num_labels
         A			:  Union[str, Any]   =		TFRoFormerForTokenClassification(config=SCREAMING_SNAKE_CASE      )
         A			:  Any   =		{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         A			:  Optional[int]   =		model(SCREAMING_SNAKE_CASE      )
         self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.seq_length, self.num_labels)      )





     def 							__lowerCAmelCase				(				self		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )					->							Optional[Any]:






         """simple docstring"""






         A			:  List[str]   =		TFRoFormerForQuestionAnswering(config=SCREAMING_SNAKE_CASE      )
         A			:  int   =		{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }

         A			:  Tuple   =		model(SCREAMING_SNAKE_CASE      )
         self.parent.assertEqual(result.start_logits.shape		,     (self.batch_size, self.seq_length)      )
         self.parent.assertEqual(result.end_logits.shape		,     (self.batch_size, self.seq_length)      )





     def 							__lowerCAmelCase				(				self      )					->							str:






         """simple docstring"""






         A			:  List[Any]   =		self.prepare_config_and_inputs()
         (
             (
             A						
         ),    (
             A						
         ),    (
             A						
         ),    (
             A						
         ),    (
             A						
         ),    (
             A						
         ),    (
             A						
         ),    						
         )			:  Dict   =		config_and_inputs
         A			:  Tuple   =		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict





@require_tf
class 					A							(				__snake_case       ,     __snake_case       ,     unittest.TestCase		):
     __magic_name__          =						(
         (
             TFRoFormerModel,
             TFRoFormerForCausalLM,
             TFRoFormerForMaskedLM,
             TFRoFormerForQuestionAnswering,
             TFRoFormerForSequenceClassification,
             TFRoFormerForTokenClassification,
             TFRoFormerForMultipleChoice,
         )
         if is_tf_available()
         else ()
     )
     __magic_name__          =						(
         {
             "feature-extraction": TFRoFormerModel,
             "fill-mask": TFRoFormerForMaskedLM,
             "question-answering": TFRoFormerForQuestionAnswering,
             "text-classification": TFRoFormerForSequenceClassification,
             "text-generation": TFRoFormerForCausalLM,
             "token-classification": TFRoFormerForTokenClassification,
             "zero-shot": TFRoFormerForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )

     __magic_name__          =						False
     __magic_name__          =						False





     def 							__lowerCAmelCase				(				self		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )					->							Tuple:






         """simple docstring"""






         if pipeline_test_casse_name == "TextGenerationPipelineTests":
             return True

         return False





     def 							__lowerCAmelCase				(				self      )					->							int:






         """simple docstring"""






         A			:  str   =		TFRoFormerModelTester(self      )
         A			:  Any   =		ConfigTester(self		,     config_class=SCREAMING_SNAKE_CASE		,     hidden_size=37      )





     def 							__lowerCAmelCase				(				self      )					->							str:






         """simple docstring"""






         self.config_tester.run_common_tests()





     def 							__lowerCAmelCase				(				self      )					->							Dict:






         """simple docstring"""






         A			:  Dict   =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE      )





     def 							__lowerCAmelCase				(				self      )					->							Union[str, Any]:






         """simple docstring"""






         A			:  int   =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE      )





     def 							__lowerCAmelCase				(				self      )					->							Tuple:






         """simple docstring"""






         A			:  int   =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_lm_head(*SCREAMING_SNAKE_CASE      )





     def 							__lowerCAmelCase				(				self      )					->							str:






         """simple docstring"""






         A			:  str   =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE      )





     def 							__lowerCAmelCase				(				self      )					->							Tuple:






         """simple docstring"""






         A			:  Dict   =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE      )





     def 							__lowerCAmelCase				(				self      )					->							Optional[Any]:






         """simple docstring"""






         A			:  Dict   =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE      )





     def 							__lowerCAmelCase				(				self      )					->							Optional[Any]:






         """simple docstring"""






         A			:  Union[str, Any]   =		self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE      )





     @slow
     def 							__lowerCAmelCase				(				self      )					->							Union[str, Any]:






         """simple docstring"""






         A			:  Union[str, Any]   =		TFRoFormerModel.from_pretrained('''junnyu/roformer_chinese_base'''      )
         self.assertIsNotNone(SCREAMING_SNAKE_CASE      )





@require_tf
class 					A							(				unittest.TestCase		):





     @slow
     def 							__lowerCAmelCase				(				self      )					->							List[str]:






         """simple docstring"""






         A			:  Union[str, Any]   =		TFRoFormerForMaskedLM.from_pretrained('''junnyu/roformer_chinese_base'''      )
         A			:  Union[str, Any]   =		tf.constant([[0, 1, 2, 3, 4, 5]]      )
         A			:  Dict   =		model(SCREAMING_SNAKE_CASE      )[0]

         # TODO Replace vocab size
         A			:  Optional[int]   =		50000

         A			:  List[Any]   =		[1, 6, vocab_size]
         self.assertEqual(output.shape		,     SCREAMING_SNAKE_CASE      )

         print(output[:, :3, :3]      )

         # TODO Replace values below with what was printed above.
         A			:  Any   =		tf.constant(
             [
                 [
                     [-0.12_053_341, -1.0_264_901, 0.29_221_946],
                     [-1.5_133_783, 0.197_433, 0.15_190_607],
                     [-5.0_135_403, -3.900_256, -0.84_038_764],
                 ]
             ]      )
         tf.debugging.assert_near(output[:, :3, :3]		,     SCREAMING_SNAKE_CASE		,     atol=1e-4      )





@require_tf
class 					A							(				unittest.TestCase		):
     __magic_name__          =						1E-4





     def 							__lowerCAmelCase				(				self      )					->							Union[str, Any]:






         """simple docstring"""






         A			:  Union[str, Any]   =		tf.constant([[4, 10]]      )
         A			:  List[str]   =		TFRoFormerSinusoidalPositionalEmbedding(num_positions=6		,     embedding_dim=6      )

         A			:  List[Any]   =		emba(input_ids.shape      )
         A			:  Dict   =		tf.constant(
             [[0.0_000, 0.0_000, 0.0_000, 1.0_000, 1.0_000, 1.0_000], [0.8_415, 0.0_464, 0.0_022, 0.5_403, 0.9_989, 1.0_000]]      )

         tf.debugging.assert_near(SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     atol=self.tolerance      )





     def 							__lowerCAmelCase				(				self      )					->							Dict:






         """simple docstring"""






         A			:  Tuple   =		tf.constant(
             [
                 [0.0_000, 0.0_000, 0.0_000, 0.0_000, 0.0_000],
                 [0.8_415, 0.8_219, 0.8_020, 0.7_819, 0.7_617],
                 [0.9_093, 0.9_364, 0.9_581, 0.9_749, 0.9_870],
             ]      )
         A			:  Tuple   =		TFRoFormerSinusoidalPositionalEmbedding(num_positions=512		,     embedding_dim=512      )
         emba([2, 16, 512]      )
         A			:  Dict   =		emba.weight[:3, :5]

         tf.debugging.assert_near(SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     atol=self.tolerance      )





@require_tf
class 					A							(				unittest.TestCase		):
     __magic_name__          =						1E-4





     def 							__lowerCAmelCase				(				self      )					->							Union[str, Any]:






         """simple docstring"""






         A			:  str   =		tf.reshape(tf.range(2 * 12 * 16 * 64		,     dtype=tf.floataa      )		,     shape=(2, 12, 16, 64)      ) / 100

         A			:  Optional[Any]   =		-tf.reshape(tf.range(2 * 12 * 16 * 64		,     dtype=tf.floataa      )		,     shape=(2, 12, 16, 64)      ) / 100

         A			:  int   =		TFRoFormerSinusoidalPositionalEmbedding(num_positions=32		,     embedding_dim=64      )
         A			:  Dict   =		embed_positions([2, 16, 768]      )[None, None, :, :]

         A,    A			:  List[Any]   =		TFRoFormerSelfAttention.apply_rotary_position_embeddings(
             SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE		,     SCREAMING_SNAKE_CASE      )

         A			:  List[Any]   =		tf.constant(
             [
                 [0.0_000, 0.0_100, 0.0_200, 0.0_300, 0.0_400, 0.0_500, 0.0_600, 0.0_700],
                 [-0.2_012, 0.8_897, 0.0_263, 0.9_401, 0.2_074, 0.9_463, 0.3_481, 0.9_343],
                 [-1.7_057, 0.6_271, -1.2_145, 1.3_897, -0.6_303, 1.7_647, -0.1_173, 1.8_985],
                 [-2.1_731, -1.6_397, -2.7_358, 0.2_854, -2.1_840, 1.7_183, -1.3_018, 2.4_871],
                 [0.2_717, -3.6_173, -2.9_206, -2.1_988, -3.6_638, 0.3_858, -2.9_155, 2.2_980],
                 [3.9_859, -2.1_580, -0.7_984, -4.4_904, -4.1_181, -2.0_252, -4.4_782, 1.1_253],
             ]      )
         A			:  Union[str, Any]   =		tf.constant(
             [
                 [0.0_000, -0.0_100, -0.0_200, -0.0_300, -0.0_400, -0.0_500, -0.0_600, -0.0_700],
                 [0.2_012, -0.8_897, -0.0_263, -0.9_401, -0.2_074, -0.9_463, -0.3_481, -0.9_343],
                 [1.7_057, -0.6_271, 1.2_145, -1.3_897, 0.6_303, -1.7_647, 0.1_173, -1.8_985],
                 [2.1_731, 1.6_397, 2.7_358, -0.2_854, 2.1_840, -1.7_183, 1.3_018, -2.4_871],
                 [-0.2_717, 3.6_173, 2.9_206, 2.1_988, 3.6_638, -0.3_858, 2.9_155, -2.2_980],
                 [-3.9_859, 2.1_580, 0.7_984, 4.4_904, 4.1_181, 2.0_252, 4.4_782, -1.1_253],
             ]      )

         tf.debugging.assert_near(query_layer[0, 0, :6, :8]		,     SCREAMING_SNAKE_CASE		,     atol=self.tolerance      )
         tf.debugging.assert_near(key_layer[0, 0, :6, :8]		,     SCREAMING_SNAKE_CASE		,     atol=self.tolerance      )









def 	_lowerCAmelCase (    A__ = 50_000_000						):
      lowercase__  =	set()
      lowercase__  =	int((limit - 24) ** (1 / 2)						)

      lowercase__  =	set(range(3    ,	prime_square_limit + 1    ,	2						)						)
      primes.add(2						)
      for p in range(3    ,	prime_square_limit + 1    ,	2						):
            if p not in primes:
                  continue
            primes.difference_update(set(range(p * p    ,	prime_square_limit + 1    ,	A__						)						)						)

      for primea in primes:
            lowercase__  =	primea * primea
            for primea in primes:
                  lowercase__  =	primea * primea * primea
                  if square + cube >= limit - 16:
                        break
                  for primea in primes:
                        lowercase__  =	primea * primea * primea * primea
                        lowercase__  =	square + cube + tetr
                        if total >= limit:
                              break
                        ret.add(A__						)

      return len(A__						)


if __name__ == "__main__":
       print(F'''{solution() = }''')
