





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
   import torch


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


lowerCAmelCase_				= logging.get_logger(__name__)
lowerCAmelCase_				= {'vocab_file': 'spiece.model'}

lowerCAmelCase_				= {
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

lowerCAmelCase_				= {
    'AI-Sweden/gpt-sw3-126m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-350m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-1.6b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-6.7b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-20b': 2_0_4_8,
}

class 					A     (__UpperCAmelCase					):
      _SCREAMING_SNAKE_CASE       =				VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE       =				PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE       =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _SCREAMING_SNAKE_CASE       =				['''input_ids''', '''attention_mask''']



      def __init__(       self  ,	lowercase_  ,	lowercase_=False  ,	lowercase_=False  ,	lowercase_=False  ,	lowercase_=None  ,	lowercase_=None  ,	lowercase_=None  ,	lowercase_=None  ,	lowercase_ = None  ,	**lowercase_  ,	)   -> None:



             '''simple docstring'''

             _snake_case   :				Union[str, Any]  					=    {} if sp_model_kwargs is None else sp_model_kwargs

             _snake_case   :				Union[str, Any]  					=    kwargs.get('''name_or_path'''       )
             if name_or_path is None:
                    logger.warning(
                        '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
                        ''' you are testing the model, this can safely be ignored'''       )
                    _snake_case   :				Dict  					=    '''None'''

             # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
             _snake_case   :				Any  					=    '''<|endoftext|>''' if eos_token is None else eos_token
             _snake_case   :				Any  					=    '''<unk>''' if unk_token is None else unk_token
             if "gpt-sw3-7b" in name_or_path:
                    _snake_case   :				List[str]  					=    unk_token if pad_token is None else pad_token
                    _snake_case   :				int  					=    eos_token if bos_token is None else bos_token
             else:
                    _snake_case   :				int  					=    '''<pad>''' if pad_token is None else pad_token
                    _snake_case   :				Optional[Any]  					=    '''<s>''' if bos_token is None else bos_token

             super().__init__(
                 do_lower_case=lowerCamelCase_  ,	remove_space=lowerCamelCase_  ,	keep_accents=lowerCamelCase_  ,	bos_token=lowerCamelCase_  ,	eos_token=lowerCamelCase_  ,	unk_token=lowerCamelCase_  ,	pad_token=lowerCamelCase_  ,	sp_model_kwargs=self.sp_model_kwargs  ,	**lowerCamelCase_  ,	)

             _snake_case   :				Union[str, Any]  					=    do_lower_case
             _snake_case   :				Optional[Any]  					=    remove_space
             _snake_case   :				List[str]  					=    keep_accents
             _snake_case   :				Tuple  					=    vocab_file

             _snake_case   :				Any  					=    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
             self.sp_model.Load(lowerCamelCase_       )

             # Used for whitespace normalization in input texts
             # fmt : off
             _snake_case   :				List[str]  					=    {''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
             # fmt : on

             # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
             _snake_case   :				Optional[int]  					=    re.compile(
                 F'''[{''.join(map(lowerCamelCase_  ,	list(range(0  ,	9       )       ) + list(range(11  ,	32       )       ) + list(range(127  ,	160       )       ) + [160, 173, 8203]       )       )}]'''       )



      def __getstate__(       self       )   -> List[str]:



             '''simple docstring'''

             _snake_case   :				List[Any]  					=    self.__dict__.copy()
             _snake_case   :				List[str]  					=    None
             return state



      def __setstate__(       self  ,	lowercase_       )   -> Union[str, Any]:



             '''simple docstring'''

             _snake_case   :				Optional[int]  					=    d

             # for backward compatibility
             if not hasattr(self  ,	'''sp_model_kwargs'''       ):
                    _snake_case   :				str  					=    {}

             _snake_case   :				Any  					=    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
             self.sp_model.Load(self.vocab_file       )



      @property
      # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
      def  __a			(       self       )   -> int:



             '''simple docstring'''

             return len(self.sp_model       )



      def  __a			(       self  ,	lowercase_       )   -> str:



             '''simple docstring'''

             _snake_case   :				int  					=    self.non_printing_characters_re.sub(''''''  ,	lowerCamelCase_       )

             # Normalize whitespaces
             _snake_case   :				Optional[Any]  					=    ''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]       )

             # NFC Unicode normalization
             _snake_case   :				Any  					=    unicodedata.normalize('''NFC'''  ,	lowerCamelCase_       )
             return text



      def  __a			(       self  ,	lowercase_  ,	**lowercase_       )   -> List[str]:



             '''simple docstring'''

             _snake_case   :				Any  					=    self.preprocess_text(lowerCamelCase_       )
             return self.sp_model.encode(lowerCamelCase_  ,	out_type=lowerCamelCase_       )



      def  __a			(       self  ,	lowercase_       )   -> int:



             '''simple docstring'''

             return self.sp_model.PieceToId(lowerCamelCase_       )



      def  __a			(       self  ,	lowercase_       )   -> str:



             '''simple docstring'''

             return self.sp_model.IdToPiece(lowerCamelCase_       )



      @staticmethod
      def  __a			(       lowercase_       )   -> str:



             '''simple docstring'''

             return out_string



      def  __a			(       self  ,	lowercase_       )   -> str:



             '''simple docstring'''

             _snake_case   :				Optional[Any]  					=    []
             _snake_case   :				Optional[int]  					=    ''''''
             _snake_case   :				Dict  					=    False
             for token in tokens:
                    # make sure that special tokens are not decoded using sentencepiece model
                    if token in self.all_special_tokens:
                           # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                           if not prev_is_special:
                                  out_string += " "

                           out_string += self.sp_model.decode(lowerCamelCase_       ) + token
                           _snake_case   :				List[Any]  					=    True
                           _snake_case   :				Optional[Any]  					=    []
                    else:
                           current_sub_tokens.append(lowerCamelCase_       )
                           _snake_case   :				Optional[int]  					=    False
             out_string += self.sp_model.decode(lowerCamelCase_       )

             return out_string



      def  __a			(       self       )   -> Dict[str, int]:



             '''simple docstring'''

             _snake_case   :				List[Any]  					=    {self.convert_ids_to_tokens(lowerCamelCase_       ): i for i in range(self.vocab_size       )}
             vocab.update(self.added_tokens_encoder       )
             return vocab



      def  __a			(       self  ,	lowercase_  ,	lowercase_ = None       )   -> Tuple[str]:



             '''simple docstring'''

             if not os.path.isdir(lowerCamelCase_       ):
                    logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
                    return
             _snake_case   :				Union[str, Any]  					=    os.path.join(
                 lowerCamelCase_  ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

             if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowerCamelCase_       ) and os.path.isfile(self.vocab_file       ):
                    copyfile(self.vocab_file  ,	lowerCamelCase_       )
             elif not os.path.isfile(self.vocab_file       ):
                    with open(lowerCamelCase_  ,	'''wb'''       ) as fi:
                           _snake_case   :				str  					=    self.sp_model.serialized_model_proto()
                           fi.write(lowerCamelCase_       )

             return (out_vocab_file,)



      def  __a			(       self  ,	lowercase_  ,	lowercase_ = False       )   -> Union[List[int], List[List[int]], "torch.Tensor"]:



             '''simple docstring'''

             if isinstance(lowerCamelCase_  ,	lowerCamelCase_       ):
                    _snake_case   :				Dict  					=    self.preprocess_text(lowerCamelCase_       )
                    _snake_case   :				List[Any]  					=    self.sp_model.encode(lowerCamelCase_       )
             else:
                    _snake_case   :				Optional[int]  					=    [self.preprocess_text(lowerCamelCase_       ) for t in text]
                    _snake_case   :				List[Any]  					=    self.sp_model.encode(lowerCamelCase_       )

             if return_tensors is True or return_tensors == "pt":
                    _snake_case   :				Optional[Any]  					=    torch.tensor(lowerCamelCase_       )

             return token_ids



      def  __a			(       self  ,	lowercase_       )   -> str:



             '''simple docstring'''

             return self.sp_model.decode(lowerCamelCase_       )



      def  __a			(       self  ,	lowercase_       )   -> List[int]:



             '''simple docstring'''

             _snake_case   :				Any  					=    [F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
             _snake_case   :				Optional[Any]  					=    (
                 F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(lowerCamelCase_       ) + F'''{self.bos_token}Bot:'''
             )
             return self.encode(text=lowerCamelCase_       )







'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TFRoFormerForCausalLM,
      TFRoFormerForMaskedLM,
      TFRoFormerForMultipleChoice,
      TFRoFormerForQuestionAnswering,
      TFRoFormerForSequenceClassification,
      TFRoFormerForTokenClassification,
      TFRoFormerModel,
  )
  from transformers.models.roformer.modeling_tf_roformer import (
      TFRoFormerSelfAttention,
      TFRoFormerSinusoidalPositionalEmbedding,
  )


class 					_lowercase						:
      '''simple docstring'''


      def __init__(					self       ,lowerCamelCase_       ,lowerCamelCase_=13       ,lowerCamelCase_=7       ,lowerCamelCase_=True       ,lowerCamelCase_=True       ,lowerCamelCase_=True       ,lowerCamelCase_=True       ,lowerCamelCase_=99       ,lowerCamelCase_=32       ,lowerCamelCase_=2       ,lowerCamelCase_=4       ,lowerCamelCase_=37       ,lowerCamelCase_="gelu"       ,lowerCamelCase_=0.1       ,lowerCamelCase_=0.1       ,lowerCamelCase_=512       ,lowerCamelCase_=16       ,lowerCamelCase_=2       ,lowerCamelCase_=0.02       ,lowerCamelCase_=3       ,lowerCamelCase_=4       ,lowerCamelCase_=None       ,)      ->      str:



         '''simple docstring'''



         UpperCAmelCase__				:					Any 					=	parent
         UpperCAmelCase__				:					str 					=	13
         UpperCAmelCase__				:					Any 					=	7
         UpperCAmelCase__				:					str 					=	True
         UpperCAmelCase__				:					List[Any] 					=	True
         UpperCAmelCase__				:					Tuple 					=	True
         UpperCAmelCase__				:					List[str] 					=	True
         UpperCAmelCase__				:					List[Any] 					=	99
         UpperCAmelCase__				:					str 					=	32
         UpperCAmelCase__				:					Dict 					=	2
         UpperCAmelCase__				:					Union[str, Any] 					=	4
         UpperCAmelCase__				:					Dict 					=	37
         UpperCAmelCase__				:					Dict 					=	'''gelu'''
         UpperCAmelCase__				:					List[str] 					=	0.1
         UpperCAmelCase__				:					List[str] 					=	0.1
         UpperCAmelCase__				:					Optional[int] 					=	512
         UpperCAmelCase__				:					Any 					=	16
         UpperCAmelCase__				:					List[Any] 					=	2
         UpperCAmelCase__				:					Optional[Any] 					=	0.02
         UpperCAmelCase__				:					Optional[int] 					=	3
         UpperCAmelCase__				:					Optional[int] 					=	4
         UpperCAmelCase__				:					Optional[int] 					=	None




      def 		lowerCAmelCase__       (					self				)      ->      Optional[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	ids_tensor([self.batch_size, self.seq_length]       ,self.vocab_size				)

         UpperCAmelCase__				:					List[str] 					=	None
         if self.use_input_mask:
            UpperCAmelCase__				:					int 					=	random_attention_mask([self.batch_size, self.seq_length]				)

         UpperCAmelCase__				:					str 					=	None
         if self.use_token_type_ids:
            UpperCAmelCase__				:					Dict 					=	ids_tensor([self.batch_size, self.seq_length]       ,self.type_vocab_size				)

         UpperCAmelCase__				:					List[str] 					=	None
         UpperCAmelCase__				:					Any 					=	None
         UpperCAmelCase__				:					Dict 					=	None
         if self.use_labels:
            UpperCAmelCase__				:					Optional[Any] 					=	ids_tensor([self.batch_size]       ,self.type_sequence_label_size				)
            UpperCAmelCase__				:					Dict 					=	ids_tensor([self.batch_size, self.seq_length]       ,self.num_labels				)
            UpperCAmelCase__				:					List[str] 					=	ids_tensor([self.batch_size]       ,self.num_choices				)

         UpperCAmelCase__				:					Union[str, Any] 					=	RoFormerConfig(
             vocab_size=self.vocab_size       ,hidden_size=self.hidden_size       ,num_hidden_layers=self.num_hidden_layers       ,num_attention_heads=self.num_attention_heads       ,intermediate_size=self.intermediate_size       ,hidden_act=self.hidden_act       ,hidden_dropout_prob=self.hidden_dropout_prob       ,attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,max_position_embeddings=self.max_position_embeddings       ,type_vocab_size=self.type_vocab_size       ,initializer_range=self.initializer_range       ,return_dict=lowerCamelCase_       ,)

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      int:



         '''simple docstring'''



         UpperCAmelCase__				:					List[str] 					=	TFRoFormerModel(config=lowerCamelCase_				)
         UpperCAmelCase__				:					str 					=	{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

         UpperCAmelCase__				:					str 					=	[input_ids, input_mask]
         UpperCAmelCase__				:					Any 					=	model(lowerCamelCase_				)

         UpperCAmelCase__				:					List[Any] 					=	model(lowerCamelCase_				)

         self.parent.assertEqual(result.last_hidden_state.shape       ,(self.batch_size, self.seq_length, self.hidden_size)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Tuple:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	True
         UpperCAmelCase__				:					Any 					=	TFRoFormerForCausalLM(config=lowerCamelCase_				)
         UpperCAmelCase__				:					List[str] 					=	{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         UpperCAmelCase__				:					int 					=	model(lowerCamelCase_				)['''logits''']
         self.parent.assertListEqual(
             list(prediction_scores.numpy().shape				)       ,[self.batch_size, self.seq_length, self.vocab_size]				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      List[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					Union[str, Any] 					=	TFRoFormerForMaskedLM(config=lowerCamelCase_				)
         UpperCAmelCase__				:					Dict 					=	{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         UpperCAmelCase__				:					Dict 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.seq_length, self.vocab_size)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	self.num_labels
         UpperCAmelCase__				:					List[Any] 					=	TFRoFormerForSequenceClassification(config=lowerCamelCase_				)
         UpperCAmelCase__				:					List[Any] 					=	{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }

         UpperCAmelCase__				:					Union[str, Any] 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.num_labels)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Dict:



         '''simple docstring'''



         UpperCAmelCase__				:					int 					=	self.num_choices
         UpperCAmelCase__				:					Optional[int] 					=	TFRoFormerForMultipleChoice(config=lowerCamelCase_				)
         UpperCAmelCase__				:					Tuple 					=	tf.tile(tf.expand_dims(lowerCamelCase_       ,1				)       ,(1, self.num_choices, 1)				)
         UpperCAmelCase__				:					Optional[Any] 					=	tf.tile(tf.expand_dims(lowerCamelCase_       ,1				)       ,(1, self.num_choices, 1)				)
         UpperCAmelCase__				:					Dict 					=	tf.tile(tf.expand_dims(lowerCamelCase_       ,1				)       ,(1, self.num_choices, 1)				)
         UpperCAmelCase__				:					Optional[int] 					=	{
             '''input_ids''': multiple_choice_inputs_ids,
             '''attention_mask''': multiple_choice_input_mask,
             '''token_type_ids''': multiple_choice_token_type_ids,
         }
         UpperCAmelCase__				:					str 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.num_choices)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Any:



         '''simple docstring'''



         UpperCAmelCase__				:					Dict 					=	self.num_labels
         UpperCAmelCase__				:					Optional[Any] 					=	TFRoFormerForTokenClassification(config=lowerCamelCase_				)
         UpperCAmelCase__				:					Dict 					=	{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }
         UpperCAmelCase__				:					Optional[int] 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.seq_length, self.num_labels)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Optional[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	TFRoFormerForQuestionAnswering(config=lowerCamelCase_				)
         UpperCAmelCase__				:					Any 					=	{
             '''input_ids''': input_ids,
             '''attention_mask''': input_mask,
             '''token_type_ids''': token_type_ids,
         }

         UpperCAmelCase__				:					int 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.start_logits.shape       ,(self.batch_size, self.seq_length)				)
         self.parent.assertEqual(result.end_logits.shape       ,(self.batch_size, self.seq_length)				)




      def 		lowerCAmelCase__       (					self				)      ->      Any:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	self.prepare_config_and_inputs()
         (
             (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   
         )				:					List[Any] 					=	config_and_inputs
         UpperCAmelCase__				:					Optional[int] 					=	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict




@require_tf
class 					_lowercase						(   lowerCAmelCase		,lowerCAmelCase		,unittest.TestCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   List[str]   				= (
          (
              TFRoFormerModel,
              TFRoFormerForCausalLM,
              TFRoFormerForMaskedLM,
              TFRoFormerForQuestionAnswering,
              TFRoFormerForSequenceClassification,
              TFRoFormerForTokenClassification,
              TFRoFormerForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      UpperCAmelCase_   :   int   				= (
          {
              '''feature-extraction''': TFRoFormerModel,
              '''fill-mask''': TFRoFormerForMaskedLM,
              '''question-answering''': TFRoFormerForQuestionAnswering,
              '''text-classification''': TFRoFormerForSequenceClassification,
              '''text-generation''': TFRoFormerForCausalLM,
              '''token-classification''': TFRoFormerForTokenClassification,
              '''zero-shot''': TFRoFormerForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )

      UpperCAmelCase_   :   Optional[int]   				= False
      UpperCAmelCase_   :   Optional[int]   				= False




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Any:



         '''simple docstring'''



         if pipeline_test_casse_name == "TextGenerationPipelineTests":
            return True

         return False




      def 		lowerCAmelCase__       (					self				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[Any] 					=	TFRoFormerModelTester(self				)
         UpperCAmelCase__				:					int 					=	ConfigTester(self       ,config_class=lowerCamelCase_       ,hidden_size=37				)




      def 		lowerCAmelCase__       (					self				)      ->      Dict:



         '''simple docstring'''



         self.config_tester.run_common_tests()




      def 		lowerCAmelCase__       (					self				)      ->      List[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Any 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      List[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					str 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_lm_head(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      Any:



         '''simple docstring'''



         UpperCAmelCase__				:					List[str] 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      str:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      List[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					str 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*lowerCamelCase_				)




      @slow
      def 		lowerCAmelCase__       (					self				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[str] 					=	TFRoFormerModel.from_pretrained('''junnyu/roformer_chinese_base'''				)
         self.assertIsNotNone(lowerCamelCase_				)




@require_tf
class 					_lowercase						(   unittest.TestCase ):
      '''simple docstring'''


      @slow
      def 		lowerCAmelCase__       (					self				)      ->      Dict:



         '''simple docstring'''



         UpperCAmelCase__				:					List[Any] 					=	TFRoFormerForMaskedLM.from_pretrained('''junnyu/roformer_chinese_base'''				)
         UpperCAmelCase__				:					Dict 					=	tf.constant([[0, 1, 2, 3, 4, 5]]				)
         UpperCAmelCase__				:					Union[str, Any] 					=	model(lowerCamelCase_				)[0]

         # TODO Replace vocab size
         UpperCAmelCase__				:					List[str] 					=	50000

         UpperCAmelCase__				:					int 					=	[1, 6, vocab_size]
         self.assertEqual(output.shape       ,lowerCamelCase_				)

         print(output[:, :3, :3]				)

         # TODO Replace values below with what was printed above.
         UpperCAmelCase__				:					Tuple 					=	tf.constant(
             [
                 [
                     [-0.1205_3341, -1.026_4901, 0.2922_1946],
                     [-1.513_3783, 0.19_7433, 0.1519_0607],
                     [-5.013_5403, -3.90_0256, -0.8403_8764],
                 ]
             ]				)
         tf.debugging.assert_near(output[:, :3, :3]       ,lowerCamelCase_       ,atol=1e-4				)




@require_tf
class 					_lowercase						(   unittest.TestCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   int   				= 1E-4




      def 		lowerCAmelCase__       (					self				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Union[str, Any] 					=	tf.constant([[4, 10]]				)
         UpperCAmelCase__				:					List[str] 					=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=6       ,embedding_dim=6				)

         UpperCAmelCase__				:					Optional[Any] 					=	emba(input_ids.shape				)
         UpperCAmelCase__				:					List[str] 					=	tf.constant(
             [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]]				)

         tf.debugging.assert_near(lowerCamelCase_       ,lowerCamelCase_       ,atol=self.tolerance				)




      def 		lowerCAmelCase__       (					self				)      ->      List[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					Any 					=	tf.constant(
             [
                 [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
                 [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
                 [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
             ]				)
         UpperCAmelCase__				:					Dict 					=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=512       ,embedding_dim=512				)
         emba([2, 16, 512]				)
         UpperCAmelCase__				:					Optional[int] 					=	emba.weight[:3, :5]

         tf.debugging.assert_near(lowerCamelCase_       ,lowerCamelCase_       ,atol=self.tolerance				)




@require_tf
class 					_lowercase						(   unittest.TestCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   Optional[int]   				= 1E-4




      def 		lowerCAmelCase__       (					self				)      ->      List[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	tf.reshape(tf.range(2 * 12 * 16 * 64       ,dtype=tf.floataa				)       ,shape=(2, 12, 16, 64)				) / 100

         UpperCAmelCase__				:					int 					=	-tf.reshape(tf.range(2 * 12 * 16 * 64       ,dtype=tf.floataa				)       ,shape=(2, 12, 16, 64)				) / 100

         UpperCAmelCase__				:					Optional[Any] 					=	TFRoFormerSinusoidalPositionalEmbedding(num_positions=32       ,embedding_dim=64				)
         UpperCAmelCase__				:					int 					=	embed_positions([2, 16, 768]				)[None, None, :, :]

         UpperCAmelCase__					,   UpperCAmelCase__				:					Dict 					=	TFRoFormerSelfAttention.apply_rotary_position_embeddings(
             lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)

         UpperCAmelCase__				:					Any 					=	tf.constant(
             [
                 [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
                 [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
                 [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
                 [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
                 [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
                 [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
             ]				)
         UpperCAmelCase__				:					List[str] 					=	tf.constant(
             [
                 [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
                 [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
                 [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
                 [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
                 [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
                 [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
             ]				)

         tf.debugging.assert_near(query_layer[0, 0, :6, :8]       ,lowerCamelCase_       ,atol=self.tolerance				)
         tf.debugging.assert_near(key_layer[0, 0, :6, :8]       ,lowerCamelCase_       ,atol=self.tolerance				)




