'''simple docstring'''

import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      AlbertForMaskedLM,
      AlbertForMultipleChoice,
      AlbertForPreTraining,
      AlbertForQuestionAnswering,
      AlbertForSequenceClassification,
      AlbertForTokenClassification,
      AlbertModel,
  )
  from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST


class 					_lowercase						:
      '''simple docstring'''


      def __init__(					self       ,lowerCamelCase_       ,lowerCamelCase_=13       ,lowerCamelCase_=7       ,lowerCamelCase_=True       ,lowerCamelCase_=True       ,lowerCamelCase_=True       ,lowerCamelCase_=True       ,lowerCamelCase_=99       ,lowerCamelCase_=16       ,lowerCamelCase_=36       ,lowerCamelCase_=6       ,lowerCamelCase_=6       ,lowerCamelCase_=6       ,lowerCamelCase_=37       ,lowerCamelCase_="gelu"       ,lowerCamelCase_=0.1       ,lowerCamelCase_=0.1       ,lowerCamelCase_=512       ,lowerCamelCase_=16       ,lowerCamelCase_=2       ,lowerCamelCase_=0.02       ,lowerCamelCase_=3       ,lowerCamelCase_=4       ,lowerCamelCase_=None       ,)      ->      Optional[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					int 					=	parent
         UpperCAmelCase__				:					Any 					=	batch_size
         UpperCAmelCase__				:					Optional[Any] 					=	seq_length
         UpperCAmelCase__				:					Tuple 					=	is_training
         UpperCAmelCase__				:					List[Any] 					=	use_input_mask
         UpperCAmelCase__				:					Any 					=	use_token_type_ids
         UpperCAmelCase__				:					str 					=	use_labels
         UpperCAmelCase__				:					List[str] 					=	vocab_size
         UpperCAmelCase__				:					Optional[int] 					=	embedding_size
         UpperCAmelCase__				:					Union[str, Any] 					=	hidden_size
         UpperCAmelCase__				:					Optional[Any] 					=	num_hidden_layers
         UpperCAmelCase__				:					Optional[Any] 					=	num_hidden_groups
         UpperCAmelCase__				:					int 					=	num_attention_heads
         UpperCAmelCase__				:					Optional[Any] 					=	intermediate_size
         UpperCAmelCase__				:					Union[str, Any] 					=	hidden_act
         UpperCAmelCase__				:					Tuple 					=	hidden_dropout_prob
         UpperCAmelCase__				:					List[Any] 					=	attention_probs_dropout_prob
         UpperCAmelCase__				:					Dict 					=	max_position_embeddings
         UpperCAmelCase__				:					int 					=	type_vocab_size
         UpperCAmelCase__				:					Any 					=	type_sequence_label_size
         UpperCAmelCase__				:					List[str] 					=	initializer_range
         UpperCAmelCase__				:					str 					=	num_labels
         UpperCAmelCase__				:					Any 					=	num_choices
         UpperCAmelCase__				:					str 					=	scope




      def 		lowerCAmelCase__       (					self				)      ->      int:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	ids_tensor([self.batch_size, self.seq_length]       ,self.vocab_size				)

         UpperCAmelCase__				:					Optional[Any] 					=	None
         if self.use_input_mask:
            UpperCAmelCase__				:					str 					=	random_attention_mask([self.batch_size, self.seq_length]				)

         UpperCAmelCase__				:					Tuple 					=	None
         if self.use_token_type_ids:
            UpperCAmelCase__				:					Optional[Any] 					=	ids_tensor([self.batch_size, self.seq_length]       ,self.type_vocab_size				)

         UpperCAmelCase__				:					Tuple 					=	None
         UpperCAmelCase__				:					Dict 					=	None
         UpperCAmelCase__				:					int 					=	None
         if self.use_labels:
            UpperCAmelCase__				:					Union[str, Any] 					=	ids_tensor([self.batch_size]       ,self.type_sequence_label_size				)
            UpperCAmelCase__				:					Dict 					=	ids_tensor([self.batch_size, self.seq_length]       ,self.num_labels				)
            UpperCAmelCase__				:					int 					=	ids_tensor([self.batch_size]       ,self.num_choices				)

         UpperCAmelCase__				:					List[Any] 					=	self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




      def 		lowerCAmelCase__       (					self				)      ->      List[Any]:



         '''simple docstring'''



         return AlbertConfig(
             vocab_size=self.vocab_size       ,hidden_size=self.hidden_size       ,num_hidden_layers=self.num_hidden_layers       ,num_attention_heads=self.num_attention_heads       ,intermediate_size=self.intermediate_size       ,hidden_act=self.hidden_act       ,hidden_dropout_prob=self.hidden_dropout_prob       ,attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,max_position_embeddings=self.max_position_embeddings       ,type_vocab_size=self.type_vocab_size       ,initializer_range=self.initializer_range       ,num_hidden_groups=self.num_hidden_groups       ,)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Tuple:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	AlbertModel(config=lowerCamelCase_				)
         model.to(lowerCamelCase_				)
         model.eval()
         UpperCAmelCase__				:					Any 					=	model(lowerCamelCase_       ,attention_mask=lowerCamelCase_       ,token_type_ids=lowerCamelCase_				)
         UpperCAmelCase__				:					Optional[Any] 					=	model(lowerCamelCase_       ,token_type_ids=lowerCamelCase_				)
         UpperCAmelCase__				:					List[Any] 					=	model(lowerCamelCase_				)
         self.parent.assertEqual(result.last_hidden_state.shape       ,(self.batch_size, self.seq_length, self.hidden_size)				)
         self.parent.assertEqual(result.pooler_output.shape       ,(self.batch_size, self.hidden_size)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      List[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					int 					=	AlbertForPreTraining(config=lowerCamelCase_				)
         model.to(lowerCamelCase_				)
         model.eval()
         UpperCAmelCase__				:					Dict 					=	model(
             lowerCamelCase_       ,attention_mask=lowerCamelCase_       ,token_type_ids=lowerCamelCase_       ,labels=lowerCamelCase_       ,sentence_order_label=lowerCamelCase_       ,)
         self.parent.assertEqual(result.prediction_logits.shape       ,(self.batch_size, self.seq_length, self.vocab_size)				)
         self.parent.assertEqual(result.sop_logits.shape       ,(self.batch_size, config.num_labels)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Any:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	AlbertForMaskedLM(config=lowerCamelCase_				)
         model.to(lowerCamelCase_				)
         model.eval()
         UpperCAmelCase__				:					Optional[Any] 					=	model(lowerCamelCase_       ,attention_mask=lowerCamelCase_       ,token_type_ids=lowerCamelCase_       ,labels=lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.seq_length, self.vocab_size)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Tuple:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	AlbertForQuestionAnswering(config=lowerCamelCase_				)
         model.to(lowerCamelCase_				)
         model.eval()
         UpperCAmelCase__				:					Any 					=	model(
             lowerCamelCase_       ,attention_mask=lowerCamelCase_       ,token_type_ids=lowerCamelCase_       ,start_positions=lowerCamelCase_       ,end_positions=lowerCamelCase_       ,)
         self.parent.assertEqual(result.start_logits.shape       ,(self.batch_size, self.seq_length)				)
         self.parent.assertEqual(result.end_logits.shape       ,(self.batch_size, self.seq_length)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Optional[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					Any 					=	self.num_labels
         UpperCAmelCase__				:					Any 					=	AlbertForSequenceClassification(lowerCamelCase_				)
         model.to(lowerCamelCase_				)
         model.eval()
         UpperCAmelCase__				:					Tuple 					=	model(lowerCamelCase_       ,attention_mask=lowerCamelCase_       ,token_type_ids=lowerCamelCase_       ,labels=lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.num_labels)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      Dict:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	self.num_labels
         UpperCAmelCase__				:					Tuple 					=	AlbertForTokenClassification(config=lowerCamelCase_				)
         model.to(lowerCamelCase_				)
         model.eval()
         UpperCAmelCase__				:					Any 					=	model(lowerCamelCase_       ,attention_mask=lowerCamelCase_       ,token_type_ids=lowerCamelCase_       ,labels=lowerCamelCase_				)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.seq_length, self.num_labels)				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_				)      ->      List[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	self.num_choices
         UpperCAmelCase__				:					List[Any] 					=	AlbertForMultipleChoice(config=lowerCamelCase_				)
         model.to(lowerCamelCase_				)
         model.eval()
         UpperCAmelCase__				:					int 					=	input_ids.unsqueeze(1				).expand(-1       ,self.num_choices       ,-1				).contiguous()
         UpperCAmelCase__				:					Union[str, Any] 					=	token_type_ids.unsqueeze(1				).expand(-1       ,self.num_choices       ,-1				).contiguous()
         UpperCAmelCase__				:					List[str] 					=	input_mask.unsqueeze(1				).expand(-1       ,self.num_choices       ,-1				).contiguous()
         UpperCAmelCase__				:					Optional[Any] 					=	model(
             lowerCamelCase_       ,attention_mask=lowerCamelCase_       ,token_type_ids=lowerCamelCase_       ,labels=lowerCamelCase_       ,)
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.num_choices)				)




      def 		lowerCAmelCase__       (					self				)      ->      List[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					Any 					=	self.prepare_config_and_inputs()
         (
             (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   (
             UpperCAmelCase__
         )					,   
         )				:					Dict 					=	config_and_inputs
         UpperCAmelCase__				:					List[str] 					=	{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
         return config, inputs_dict




@require_torch
class 					_lowercase						(   lowerCAmelCase		,lowerCAmelCase		,unittest.TestCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   List[str]   				= (
          (
              AlbertModel,
              AlbertForPreTraining,
              AlbertForMaskedLM,
              AlbertForMultipleChoice,
              AlbertForSequenceClassification,
              AlbertForTokenClassification,
              AlbertForQuestionAnswering,
          )
          if is_torch_available()
          else ()
      )
      UpperCAmelCase_   :   Union[str, Any]   				= (
          {
              '''feature-extraction''': AlbertModel,
              '''fill-mask''': AlbertForMaskedLM,
              '''question-answering''': AlbertForQuestionAnswering,
              '''text-classification''': AlbertForSequenceClassification,
              '''token-classification''': AlbertForTokenClassification,
              '''zero-shot''': AlbertForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      UpperCAmelCase_   :   Union[str, Any]   				= True




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_       ,lowerCamelCase_=False				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Any 					=	super()._prepare_for_class(lowerCamelCase_       ,lowerCamelCase_       ,return_labels=lowerCamelCase_				)

         if return_labels:
            if model_class in get_values(lowerCamelCase_				):
               UpperCAmelCase__				:					str 					=	torch.zeros(
                   (self.model_tester.batch_size, self.model_tester.seq_length)       ,dtype=torch.long       ,device=lowerCamelCase_				)
               UpperCAmelCase__				:					str 					=	torch.zeros(
                   self.model_tester.batch_size       ,dtype=torch.long       ,device=lowerCamelCase_				)
         return inputs_dict




      def 		lowerCAmelCase__       (					self				)      ->      Optional[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	AlbertModelTester(self				)
         UpperCAmelCase__				:					List[str] 					=	ConfigTester(self       ,config_class=lowerCamelCase_       ,hidden_size=37				)




      def 		lowerCAmelCase__       (					self				)      ->      int:



         '''simple docstring'''



         self.config_tester.run_common_tests()




      def 		lowerCAmelCase__       (					self				)      ->      Dict:



         '''simple docstring'''



         UpperCAmelCase__				:					Dict 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      str:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_pretraining(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      Any:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      List[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					int 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      Any:



         '''simple docstring'''



         UpperCAmelCase__				:					List[str] 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      int:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase_				)




      def 		lowerCAmelCase__       (					self				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					int 					=	self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
            UpperCAmelCase__				:					List[str] 					=	type
            self.model_tester.create_and_check_model(*lowerCamelCase_				)




      @slow
      def 		lowerCAmelCase__       (					self				)      ->      Dict:



         '''simple docstring'''



         for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            UpperCAmelCase__				:					Dict 					=	AlbertModel.from_pretrained(lowerCamelCase_				)
            self.assertIsNotNone(lowerCamelCase_				)




@require_torch
class 					_lowercase						(   unittest.TestCase ):
      '''simple docstring'''


      @slow
      def 		lowerCAmelCase__       (					self				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Any 					=	AlbertModel.from_pretrained('''albert-base-v2'''				)
         UpperCAmelCase__				:					str 					=	torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]				)
         UpperCAmelCase__				:					List[str] 					=	torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]				)
         with torch.no_grad():
            UpperCAmelCase__				:					Optional[int] 					=	model(lowerCamelCase_       ,attention_mask=lowerCamelCase_				)[0]
         UpperCAmelCase__				:					Optional[Any] 					=	torch.Size((1, 11, 768)				)
         self.assertEqual(output.shape       ,lowerCamelCase_				)
         UpperCAmelCase__				:					Optional[int] 					=	torch.tensor(
             [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]				)

         self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,lowerCamelCase_       ,atol=1e-4				)				)





'''simple docstring'''

import warnings


warnings.warn(
    'memory_utils has been reorganized to utils.memory. Import `find_executable_batchsize` from the main `__init__`: '
    '`from accelerate import find_executable_batch_size` to avoid this warning.',
    FutureWarning,
)




