

import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    MODEL_FOR_PRETRAINING_MAPPING,
					    AlbertForMaskedLM,
					    AlbertForMultipleChoice,
					    AlbertForPreTraining,
					    AlbertForQuestionAnswering,
					    AlbertForSequenceClassification,
					    AlbertForTokenClassification,
					    AlbertModel,
					)
					from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class  lowercase				:




				def __init__(					self      ,   snake_case      ,   snake_case=13      ,   snake_case=7      ,   snake_case=True      ,   snake_case=True      ,   snake_case=True      ,   snake_case=True      ,   snake_case=99      ,   snake_case=16      ,   snake_case=36      ,   snake_case=6      ,   snake_case=6      ,   snake_case=6      ,   snake_case=37      ,   snake_case="gelu"      ,   snake_case=0.1      ,   snake_case=0.1      ,   snake_case=512      ,   snake_case=16      ,   snake_case=2      ,   snake_case=0.02      ,   snake_case=3      ,   snake_case=4      ,   snake_case=None      ,   ):
											snake_case_							=    parent
											snake_case_							=    batch_size
											snake_case_							=    seq_length
											snake_case_							=    is_training
											snake_case_							=    use_input_mask
											snake_case_							=    use_token_type_ids
											snake_case_							=    use_labels
											snake_case_							=    vocab_size
											snake_case_							=    embedding_size
											snake_case_							=    hidden_size
											snake_case_							=    num_hidden_layers
											snake_case_							=    num_hidden_groups
											snake_case_							=    num_attention_heads
											snake_case_							=    intermediate_size
											snake_case_							=    hidden_act
											snake_case_							=    hidden_dropout_prob
											snake_case_							=    attention_probs_dropout_prob
											snake_case_							=    max_position_embeddings
											snake_case_							=    type_vocab_size
											snake_case_							=    type_sequence_label_size
											snake_case_							=    initializer_range
											snake_case_							=    num_labels
											snake_case_							=    num_choices
											snake_case_							=    scope




				def     a							(					self       ):
											snake_case_							=    ids_tensor([self.batch_size, self.seq_length]      ,   self.vocab_size       )

											snake_case_							=    None
											if self.use_input_mask:
																		snake_case_							=    random_attention_mask([self.batch_size, self.seq_length]       )

											snake_case_							=    None
											if self.use_token_type_ids:
																		snake_case_							=    ids_tensor([self.batch_size, self.seq_length]      ,   self.type_vocab_size       )

											snake_case_							=    None
											snake_case_							=    None
											snake_case_							=    None
											if self.use_labels:
																		snake_case_							=    ids_tensor([self.batch_size]      ,   self.type_sequence_label_size       )
																		snake_case_							=    ids_tensor([self.batch_size, self.seq_length]      ,   self.num_labels       )
																		snake_case_							=    ids_tensor([self.batch_size]      ,   self.num_choices       )

											snake_case_							=    self.get_config()

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




				def     a							(					self       ):
											return AlbertConfig(
											    vocab_size=self.vocab_size      ,   hidden_size=self.hidden_size      ,   num_hidden_layers=self.num_hidden_layers      ,   num_attention_heads=self.num_attention_heads      ,   intermediate_size=self.intermediate_size      ,   hidden_act=self.hidden_act      ,   hidden_dropout_prob=self.hidden_dropout_prob      ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,   max_position_embeddings=self.max_position_embeddings      ,   type_vocab_size=self.type_vocab_size      ,   initializer_range=self.initializer_range      ,   num_hidden_groups=self.num_hidden_groups      ,   )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    AlbertModel(config=snake_case       )
											model.to(snake_case       )
											model.eval()
											snake_case_							=    model(snake_case      ,   attention_mask=snake_case      ,   token_type_ids=snake_case       )
											snake_case_							=    model(snake_case      ,   token_type_ids=snake_case       )
											snake_case_							=    model(snake_case       )
											self.parent.assertEqual(result.last_hidden_state.shape      ,   (self.batch_size, self.seq_length, self.hidden_size)       )
											self.parent.assertEqual(result.pooler_output.shape      ,   (self.batch_size, self.hidden_size)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    AlbertForPreTraining(config=snake_case       )
											model.to(snake_case       )
											model.eval()
											snake_case_							=    model(
											    snake_case      ,   attention_mask=snake_case      ,   token_type_ids=snake_case      ,   labels=snake_case      ,   sentence_order_label=snake_case      ,   )
											self.parent.assertEqual(result.prediction_logits.shape      ,   (self.batch_size, self.seq_length, self.vocab_size)       )
											self.parent.assertEqual(result.sop_logits.shape      ,   (self.batch_size, config.num_labels)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    AlbertForMaskedLM(config=snake_case       )
											model.to(snake_case       )
											model.eval()
											snake_case_							=    model(snake_case      ,   attention_mask=snake_case      ,   token_type_ids=snake_case      ,   labels=snake_case       )
											self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.seq_length, self.vocab_size)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    AlbertForQuestionAnswering(config=snake_case       )
											model.to(snake_case       )
											model.eval()
											snake_case_							=    model(
											    snake_case      ,   attention_mask=snake_case      ,   token_type_ids=snake_case      ,   start_positions=snake_case      ,   end_positions=snake_case      ,   )
											self.parent.assertEqual(result.start_logits.shape      ,   (self.batch_size, self.seq_length)       )
											self.parent.assertEqual(result.end_logits.shape      ,   (self.batch_size, self.seq_length)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    self.num_labels
											snake_case_							=    AlbertForSequenceClassification(snake_case       )
											model.to(snake_case       )
											model.eval()
											snake_case_							=    model(snake_case      ,   attention_mask=snake_case      ,   token_type_ids=snake_case      ,   labels=snake_case       )
											self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.num_labels)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    self.num_labels
											snake_case_							=    AlbertForTokenClassification(config=snake_case       )
											model.to(snake_case       )
											model.eval()
											snake_case_							=    model(snake_case      ,   attention_mask=snake_case      ,   token_type_ids=snake_case      ,   labels=snake_case       )
											self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.seq_length, self.num_labels)       )




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case      ,   snake_case       ):
											snake_case_							=    self.num_choices
											snake_case_							=    AlbertForMultipleChoice(config=snake_case       )
											model.to(snake_case       )
											model.eval()
											snake_case_							=    input_ids.unsqueeze(1       ).expand(-1      ,   self.num_choices      ,   -1       ).contiguous()
											snake_case_							=    token_type_ids.unsqueeze(1       ).expand(-1      ,   self.num_choices      ,   -1       ).contiguous()
											snake_case_							=    input_mask.unsqueeze(1       ).expand(-1      ,   self.num_choices      ,   -1       ).contiguous()
											snake_case_							=    model(
											    snake_case      ,   attention_mask=snake_case      ,   token_type_ids=snake_case      ,   labels=snake_case      ,   )
											self.parent.assertEqual(result.logits.shape      ,   (self.batch_size, self.num_choices)       )




				def     a							(					self       ):
											snake_case_							=    self.prepare_config_and_inputs()
											(
											    (
											    snake_case_
											)				,    (
											    snake_case_
											)				,    (
											    snake_case_
											)				,    (
											    snake_case_
											)				,    (
											    snake_case_
											)				,    (
											    snake_case_
											)				,    (
											    snake_case_
											)				,    
											)							=    config_and_inputs
											snake_case_							=    {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
											return config, inputs_dict







@require_torch
class  lowercase				(       lowercase_	,  lowercase_	,  unittest.TestCase			):
				__SCREAMING_SNAKE_CASE					: Any       			=			(
				    (
				        AlbertModel,
				        AlbertForPreTraining,
				        AlbertForMaskedLM,
				        AlbertForMultipleChoice,
				        AlbertForSequenceClassification,
				        AlbertForTokenClassification,
				        AlbertForQuestionAnswering,
				    )
				    if is_torch_available()
				    else ()
				)
				__SCREAMING_SNAKE_CASE					: List[str]       			=			(
				    {
				        '''feature-extraction''': AlbertModel,
				        '''fill-mask''': AlbertForMaskedLM,
				        '''question-answering''': AlbertForQuestionAnswering,
				        '''text-classification''': AlbertForSequenceClassification,
				        '''token-classification''': AlbertForTokenClassification,
				        '''zero-shot''': AlbertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				__SCREAMING_SNAKE_CASE					: Optional[int]       			=			True




				def     a							(					self      ,   snake_case      ,   snake_case      ,   snake_case=False       ):
											snake_case_							=    super()._prepare_for_class(snake_case      ,   snake_case      ,   return_labels=snake_case       )

											if return_labels:
																		if model_class in get_values(snake_case       ):
																									snake_case_							=    torch.zeros(
																									    (self.model_tester.batch_size, self.model_tester.seq_length)      ,   dtype=torch.long      ,   device=snake_case       )
																									snake_case_							=    torch.zeros(
																									    self.model_tester.batch_size      ,   dtype=torch.long      ,   device=snake_case       )
											return inputs_dict




				def     a							(					self       ):
											snake_case_							=    AlbertModelTester(self       )
											snake_case_							=    ConfigTester(self      ,   config_class=snake_case      ,   hidden_size=37       )




				def     a							(					self       ):
											self.config_tester.run_common_tests()




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*snake_case       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_pretraining(*snake_case       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*snake_case       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*snake_case       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*snake_case       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*snake_case       )




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
																		snake_case_							=    type
																		self.model_tester.create_and_check_model(*snake_case       )




				@slow
				def     a							(					self       ):
											for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		snake_case_							=    AlbertModel.from_pretrained(snake_case       )
																		self.assertIsNotNone(snake_case       )







@require_torch
class  lowercase				(       unittest.TestCase			):




				@slow
				def     a							(					self       ):
											snake_case_							=    AlbertModel.from_pretrained('albert-base-v2'       )
											snake_case_							=    torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]       )
											snake_case_							=    torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]       )
											with torch.no_grad():
																		snake_case_							=    model(snake_case      ,   attention_mask=snake_case       )[0]
											snake_case_							=    torch.Size((1, 11, 768)       )
											self.assertEqual(output.shape      ,   snake_case       )
											snake_case_							=    torch.tensor(
											    [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]]       )

											self.assertTrue(torch.allclose(output[:, 1:4, 1:4]      ,   snake_case      ,   atol=1e-4       )       )







import numpy as np





def 					__lowerCamelCase	( UpperCamelCase__      ):







							'''simple docstring'''






							return 1 / (1 + np.exp(-vector      ))





def 					__lowerCamelCase	( UpperCamelCase__      ):







							'''simple docstring'''






							return vector * sigmoid(UpperCamelCase__      )


if __name__ == "__main__":
					import doctest

					doctest.testmod()




