

import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     AlbertForMaskedLM,
     AlbertForMultipleChoice,
     AlbertForPreTraining,
     AlbertForQuestionAnswering,
     AlbertForSequenceClassification,
     AlbertForTokenClassification,
     AlbertModel,
 )
 from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST






class 						__magic_name__					:






     def __init__(				self		,						_lowercase		,						_lowercase=13		,						_lowercase=7		,						_lowercase=True		,						_lowercase=True		,						_lowercase=True		,						_lowercase=True		,						_lowercase=99		,						_lowercase=16		,						_lowercase=36		,						_lowercase=6		,						_lowercase=6		,						_lowercase=6		,						_lowercase=37		,						_lowercase="gelu"		,						_lowercase=0.1		,						_lowercase=0.1		,						_lowercase=512		,						_lowercase=16		,						_lowercase=2		,						_lowercase=0.02		,						_lowercase=3		,						_lowercase=4		,						_lowercase=None		,						)->     Tuple:
         UpperCamelCase_			  =						parent
         UpperCamelCase_			  =						batch_size
         UpperCamelCase_			  =						seq_length
         UpperCamelCase_			  =						is_training
         UpperCamelCase_			  =						use_input_mask
         UpperCamelCase_			  =						use_token_type_ids
         UpperCamelCase_			  =						use_labels
         UpperCamelCase_			  =						vocab_size
         UpperCamelCase_			  =						embedding_size
         UpperCamelCase_			  =						hidden_size
         UpperCamelCase_			  =						num_hidden_layers
         UpperCamelCase_			  =						num_hidden_groups
         UpperCamelCase_			  =						num_attention_heads
         UpperCamelCase_			  =						intermediate_size
         UpperCamelCase_			  =						hidden_act
         UpperCamelCase_			  =						hidden_dropout_prob
         UpperCamelCase_			  =						attention_probs_dropout_prob
         UpperCamelCase_			  =						max_position_embeddings
         UpperCamelCase_			  =						type_vocab_size
         UpperCamelCase_			  =						type_sequence_label_size
         UpperCamelCase_			  =						initializer_range
         UpperCamelCase_			  =						num_labels
         UpperCamelCase_			  =						num_choices
         UpperCamelCase_			  =						scope






     def 	UpperCAmelCase_				(				self	)->     Tuple:
         UpperCamelCase_			  =						ids_tensor([self.batch_size, self.seq_length]		,						self.vocab_size	)

         UpperCamelCase_			  =						None
         if self.use_input_mask:
             UpperCamelCase_			  =						random_attention_mask([self.batch_size, self.seq_length]	)

         UpperCamelCase_			  =						None
         if self.use_token_type_ids:
             UpperCamelCase_			  =						ids_tensor([self.batch_size, self.seq_length]		,						self.type_vocab_size	)

         UpperCamelCase_			  =						None
         UpperCamelCase_			  =						None
         UpperCamelCase_			  =						None
         if self.use_labels:
             UpperCamelCase_			  =						ids_tensor([self.batch_size]		,						self.type_sequence_label_size	)
             UpperCamelCase_			  =						ids_tensor([self.batch_size, self.seq_length]		,						self.num_labels	)
             UpperCamelCase_			  =						ids_tensor([self.batch_size]		,						self.num_choices	)

         UpperCamelCase_			  =						self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






     def 	UpperCAmelCase_				(				self	)->     Any:
         return AlbertConfig(
             vocab_size=self.vocab_size		,						hidden_size=self.hidden_size		,						num_hidden_layers=self.num_hidden_layers		,						num_attention_heads=self.num_attention_heads		,						intermediate_size=self.intermediate_size		,						hidden_act=self.hidden_act		,						hidden_dropout_prob=self.hidden_dropout_prob		,						attention_probs_dropout_prob=self.attention_probs_dropout_prob		,						max_position_embeddings=self.max_position_embeddings		,						type_vocab_size=self.type_vocab_size		,						initializer_range=self.initializer_range		,						num_hidden_groups=self.num_hidden_groups		,						)






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase	)->     Any:
         UpperCamelCase_			  =						AlbertModel(config=_lowercase	)
         model.to(_lowercase	)
         model.eval()
         UpperCamelCase_			  =						model(_lowercase		,						attention_mask=_lowercase		,						token_type_ids=_lowercase	)
         UpperCamelCase_			  =						model(_lowercase		,						token_type_ids=_lowercase	)
         UpperCamelCase_			  =						model(_lowercase	)
         self.parent.assertEqual(result.last_hidden_state.shape		,						(self.batch_size, self.seq_length, self.hidden_size)	)
         self.parent.assertEqual(result.pooler_output.shape		,						(self.batch_size, self.hidden_size)	)






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase	)->     Optional[int]:
         UpperCamelCase_			  =						AlbertForPreTraining(config=_lowercase	)
         model.to(_lowercase	)
         model.eval()
         UpperCamelCase_			  =						model(
             _lowercase		,						attention_mask=_lowercase		,						token_type_ids=_lowercase		,						labels=_lowercase		,						sentence_order_label=_lowercase		,						)
         self.parent.assertEqual(result.prediction_logits.shape		,						(self.batch_size, self.seq_length, self.vocab_size)	)
         self.parent.assertEqual(result.sop_logits.shape		,						(self.batch_size, config.num_labels)	)






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase	)->     Union[str, Any]:
         UpperCamelCase_			  =						AlbertForMaskedLM(config=_lowercase	)
         model.to(_lowercase	)
         model.eval()
         UpperCamelCase_			  =						model(_lowercase		,						attention_mask=_lowercase		,						token_type_ids=_lowercase		,						labels=_lowercase	)
         self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.seq_length, self.vocab_size)	)






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase	)->     Union[str, Any]:
         UpperCamelCase_			  =						AlbertForQuestionAnswering(config=_lowercase	)
         model.to(_lowercase	)
         model.eval()
         UpperCamelCase_			  =						model(
             _lowercase		,						attention_mask=_lowercase		,						token_type_ids=_lowercase		,						start_positions=_lowercase		,						end_positions=_lowercase		,						)
         self.parent.assertEqual(result.start_logits.shape		,						(self.batch_size, self.seq_length)	)
         self.parent.assertEqual(result.end_logits.shape		,						(self.batch_size, self.seq_length)	)






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase	)->     Tuple:
         UpperCamelCase_			  =						self.num_labels
         UpperCamelCase_			  =						AlbertForSequenceClassification(_lowercase	)
         model.to(_lowercase	)
         model.eval()
         UpperCamelCase_			  =						model(_lowercase		,						attention_mask=_lowercase		,						token_type_ids=_lowercase		,						labels=_lowercase	)
         self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.num_labels)	)






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase	)->     Dict:
         UpperCamelCase_			  =						self.num_labels
         UpperCamelCase_			  =						AlbertForTokenClassification(config=_lowercase	)
         model.to(_lowercase	)
         model.eval()
         UpperCamelCase_			  =						model(_lowercase		,						attention_mask=_lowercase		,						token_type_ids=_lowercase		,						labels=_lowercase	)
         self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.seq_length, self.num_labels)	)






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase		,						_lowercase	)->     Optional[Any]:
         UpperCamelCase_			  =						self.num_choices
         UpperCamelCase_			  =						AlbertForMultipleChoice(config=_lowercase	)
         model.to(_lowercase	)
         model.eval()
         UpperCamelCase_			  =						input_ids.unsqueeze(1	).expand(-1		,						self.num_choices		,						-1	).contiguous()
         UpperCamelCase_			  =						token_type_ids.unsqueeze(1	).expand(-1		,						self.num_choices		,						-1	).contiguous()
         UpperCamelCase_			  =						input_mask.unsqueeze(1	).expand(-1		,						self.num_choices		,						-1	).contiguous()
         UpperCamelCase_			  =						model(
             _lowercase		,						attention_mask=_lowercase		,						token_type_ids=_lowercase		,						labels=_lowercase		,						)
         self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.num_choices)	)






     def 	UpperCAmelCase_				(				self	)->     Dict:
         UpperCamelCase_			  =						self.prepare_config_and_inputs()
         (
             (
             UpperCamelCase_
         )  ,       (
             UpperCamelCase_
         )  ,       (
             UpperCamelCase_
         )  ,       (
             UpperCamelCase_
         )  ,       (
             UpperCamelCase_
         )  ,       (
             UpperCamelCase_
         )  ,       (
             UpperCamelCase_
         )  ,       
         )			  =						config_and_inputs
         UpperCamelCase_			  =						{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
         return config, inputs_dict






@require_torch
class 						__magic_name__					(							snake_case			,      snake_case			,      unittest.TestCase				):
     UpperCamelCase_	:Any										=      (
         (
             AlbertModel,
             AlbertForPreTraining,
             AlbertForMaskedLM,
             AlbertForMultipleChoice,
             AlbertForSequenceClassification,
             AlbertForTokenClassification,
             AlbertForQuestionAnswering,
         )
         if is_torch_available()
         else ()
     )
     UpperCamelCase_	:Tuple										=      (
         {
             """feature-extraction""": AlbertModel,
             """fill-mask""": AlbertForMaskedLM,
             """question-answering""": AlbertForQuestionAnswering,
             """text-classification""": AlbertForSequenceClassification,
             """token-classification""": AlbertForTokenClassification,
             """zero-shot""": AlbertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     UpperCamelCase_	:Tuple										=      True






     def 	UpperCAmelCase_				(				self		,						_lowercase		,						_lowercase		,						_lowercase=False	)->     List[str]:
         UpperCamelCase_			  =						super()._prepare_for_class(_lowercase		,						_lowercase		,						return_labels=_lowercase	)

         if return_labels:
             if model_class in get_values(_lowercase	):
                 UpperCamelCase_			  =						torch.zeros(
                     (self.model_tester.batch_size, self.model_tester.seq_length)		,						dtype=torch.long		,						device=_lowercase	)
                 UpperCamelCase_			  =						torch.zeros(
                     self.model_tester.batch_size		,						dtype=torch.long		,						device=_lowercase	)
         return inputs_dict






     def 	UpperCAmelCase_				(				self	)->     Optional[Any]:
         UpperCamelCase_			  =						AlbertModelTester(self	)
         UpperCamelCase_			  =						ConfigTester(self		,						config_class=_lowercase		,						hidden_size=37	)






     def 	UpperCAmelCase_				(				self	)->     List[str]:
         self.config_tester.run_common_tests()






     def 	UpperCAmelCase_				(				self	)->     str:
         UpperCamelCase_			  =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_lowercase	)






     def 	UpperCAmelCase_				(				self	)->     str:
         UpperCamelCase_			  =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_pretraining(*_lowercase	)






     def 	UpperCAmelCase_				(				self	)->     Any:
         UpperCamelCase_			  =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*_lowercase	)






     def 	UpperCAmelCase_				(				self	)->     Tuple:
         UpperCamelCase_			  =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*_lowercase	)






     def 	UpperCAmelCase_				(				self	)->     Union[str, Any]:
         UpperCamelCase_			  =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*_lowercase	)






     def 	UpperCAmelCase_				(				self	)->     List[str]:
         UpperCamelCase_			  =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*_lowercase	)






     def 	UpperCAmelCase_				(				self	)->     str:
         UpperCamelCase_			  =						self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
             UpperCamelCase_			  =						type
             self.model_tester.create_and_check_model(*_lowercase	)






     @slow
     def 	UpperCAmelCase_				(				self	)->     Union[str, Any]:
         for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             UpperCamelCase_			  =						AlbertModel.from_pretrained(_lowercase	)
             self.assertIsNotNone(_lowercase	)






@require_torch
class 						__magic_name__					(							unittest.TestCase				):






     @slow
     def 	UpperCAmelCase_				(				self	)->     Optional[Any]:
         UpperCamelCase_			  =						AlbertModel.from_pretrained("albert-base-v2"	)
         UpperCamelCase_			  =						torch.tensor([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]	)
         UpperCamelCase_			  =						torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
         with torch.no_grad():
             UpperCamelCase_			  =						model(_lowercase		,						attention_mask=_lowercase	)[0]
         UpperCamelCase_			  =						torch.Size((1, 11, 768)	)
         self.assertEqual(output.shape		,						_lowercase	)
         UpperCamelCase_			  =						torch.tensor(
             [[[-0.6_513, 1.5_035, -0.2_766], [-0.6_515, 1.5_046, -0.2_780], [-0.6_512, 1.5_049, -0.2_784]]]	)

         self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,						_lowercase		,						atol=1e-4	)	)



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE		:Optional[int]						=     {
    """configuration_rembert""": ["""REMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """RemBertConfig""", """RemBertOnnxConfig"""]
}

try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE		:str						=     ["""RemBertTokenizer"""]

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE		:Optional[Any]						=     ["""RemBertTokenizerFast"""]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE		:Dict						=     [
     """REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """RemBertForCausalLM""",
     """RemBertForMaskedLM""",
     """RemBertForMultipleChoice""",
     """RemBertForQuestionAnswering""",
     """RemBertForSequenceClassification""",
     """RemBertForTokenClassification""",
     """RemBertLayer""",
     """RemBertModel""",
     """RemBertPreTrainedModel""",
     """load_tf_weights_in_rembert""",
 ]


try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 SCREAMING_SNAKE_CASE		:Optional[int]						=     [
     """TF_REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """TFRemBertForCausalLM""",
     """TFRemBertForMaskedLM""",
     """TFRemBertForMultipleChoice""",
     """TFRemBertForQuestionAnswering""",
     """TFRemBertForSequenceClassification""",
     """TFRemBertForTokenClassification""",
     """TFRemBertLayer""",
     """TFRemBertModel""",
     """TFRemBertPreTrainedModel""",
 ]


if TYPE_CHECKING:
 from .configuration_rembert import REMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RemBertConfig, RemBertOnnxConfig

 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_rembert import RemBertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_rembert_fast import RemBertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_rembert import (
      REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      RemBertForCausalLM,
      RemBertForMaskedLM,
      RemBertForMultipleChoice,
      RemBertForQuestionAnswering,
      RemBertForSequenceClassification,
      RemBertForTokenClassification,
      RemBertLayer,
      RemBertModel,
      RemBertPreTrainedModel,
      load_tf_weights_in_rembert,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_rembert import (
      TF_REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFRemBertForCausalLM,
      TFRemBertForMaskedLM,
      TFRemBertForMultipleChoice,
      TFRemBertForQuestionAnswering,
      TFRemBertForSequenceClassification,
      TFRemBertForTokenClassification,
      TFRemBertLayer,
      TFRemBertModel,
      TFRemBertPreTrainedModel,
  )


else:
 import sys

 SCREAMING_SNAKE_CASE		:List[Any]						=     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
