





import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MODEL_FOR_PRETRAINING_MAPPING,
         AlbertForMaskedLM,
         AlbertForMultipleChoice,
         AlbertForPreTraining,
         AlbertForQuestionAnswering,
         AlbertForSequenceClassification,
         AlbertForTokenClassification,
         AlbertModel,
     )
     from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST



class 				_UpperCAmelCase    :

  """simple docstring"""


  def __init__(				self :		Dict					,  lowerCAmelCase_ :		Dict					,  lowerCAmelCase_ :		int=1_3					,  lowerCAmelCase_ :		Any=7					,  lowerCAmelCase_ :		Optional[Any]=True					,  lowerCAmelCase_ :		Tuple=True					,  lowerCAmelCase_ :		List[Any]=True					,  lowerCAmelCase_ :		Dict=True					,  lowerCAmelCase_ :		Tuple=9_9					,  lowerCAmelCase_ :		int=1_6					,  lowerCAmelCase_ :		List[str]=3_6					,  lowerCAmelCase_ :		Optional[int]=6					,  lowerCAmelCase_ :		Tuple=6					,  lowerCAmelCase_ :		Dict=6					,  lowerCAmelCase_ :		List[str]=3_7					,  lowerCAmelCase_ :		Dict="gelu"					,  lowerCAmelCase_ :		Any=0.1					,  lowerCAmelCase_ :		str=0.1					,  lowerCAmelCase_ :		List[str]=5_1_2					,  lowerCAmelCase_ :		Any=1_6					,  lowerCAmelCase_ :		List[Any]=2					,  lowerCAmelCase_ :		List[Any]=0.02					,  lowerCAmelCase_ :		str=3					,  lowerCAmelCase_ :		Dict=4					,  lowerCAmelCase_ :		Union[str, Any]=None					,  )							->						Union[str, Any]:
     __lowerCAmelCase								=	parent
     __lowerCAmelCase								=	batch_size
     __lowerCAmelCase								=	seq_length
     __lowerCAmelCase								=	is_training
     __lowerCAmelCase								=	use_input_mask
     __lowerCAmelCase								=	use_token_type_ids
     __lowerCAmelCase								=	use_labels
     __lowerCAmelCase								=	vocab_size
     __lowerCAmelCase								=	embedding_size
     __lowerCAmelCase								=	hidden_size
     __lowerCAmelCase								=	num_hidden_layers
     __lowerCAmelCase								=	num_hidden_groups
     __lowerCAmelCase								=	num_attention_heads
     __lowerCAmelCase								=	intermediate_size
     __lowerCAmelCase								=	hidden_act
     __lowerCAmelCase								=	hidden_dropout_prob
     __lowerCAmelCase								=	attention_probs_dropout_prob
     __lowerCAmelCase								=	max_position_embeddings
     __lowerCAmelCase								=	type_vocab_size
     __lowerCAmelCase								=	type_sequence_label_size
     __lowerCAmelCase								=	initializer_range
     __lowerCAmelCase								=	num_labels
     __lowerCAmelCase								=	num_choices
     __lowerCAmelCase								=	scope






  def 				lowercase		(				self :		Any     )							->						Dict:
     __lowerCAmelCase								=	ids_tensor([self.batch_size, self.seq_length]					,  self.vocab_size     )

     __lowerCAmelCase								=	None
     if self.use_input_mask:
        __lowerCAmelCase								=	random_attention_mask([self.batch_size, self.seq_length]     )

     __lowerCAmelCase								=	None
     if self.use_token_type_ids:
        __lowerCAmelCase								=	ids_tensor([self.batch_size, self.seq_length]					,  self.type_vocab_size     )

     __lowerCAmelCase								=	None
     __lowerCAmelCase								=	None
     __lowerCAmelCase								=	None
     if self.use_labels:
        __lowerCAmelCase								=	ids_tensor([self.batch_size]					,  self.type_sequence_label_size     )
        __lowerCAmelCase								=	ids_tensor([self.batch_size, self.seq_length]					,  self.num_labels     )
        __lowerCAmelCase								=	ids_tensor([self.batch_size]					,  self.num_choices     )

     __lowerCAmelCase								=	self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






  def 				lowercase		(				self :		List[Any]     )							->						Optional[Any]:
     return AlbertConfig(
         vocab_size=self.vocab_size					,  hidden_size=self.hidden_size					,  num_hidden_layers=self.num_hidden_layers					,  num_attention_heads=self.num_attention_heads					,  intermediate_size=self.intermediate_size					,  hidden_act=self.hidden_act					,  hidden_dropout_prob=self.hidden_dropout_prob					,  attention_probs_dropout_prob=self.attention_probs_dropout_prob					,  max_position_embeddings=self.max_position_embeddings					,  type_vocab_size=self.type_vocab_size					,  initializer_range=self.initializer_range					,  num_hidden_groups=self.num_hidden_groups					,  )






  def 				lowercase		(				self :		Tuple					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		Union[str, Any]					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		Optional[int]     )							->						Optional[int]:
     __lowerCAmelCase								=	AlbertModel(config=lowerCAmelCase_     )
     model.to(lowerCAmelCase_     )
     model.eval()
     __lowerCAmelCase								=	model(lowerCAmelCase_					,  attention_mask=lowerCAmelCase_					,  token_type_ids=lowerCAmelCase_     )
     __lowerCAmelCase								=	model(lowerCAmelCase_					,  token_type_ids=lowerCAmelCase_     )
     __lowerCAmelCase								=	model(lowerCAmelCase_     )
     self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.seq_length, self.hidden_size)     )
     self.parent.assertEqual(result.pooler_output.shape					,  (self.batch_size, self.hidden_size)     )






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		Optional[int]					,  lowerCAmelCase_ :		Any					,  lowerCAmelCase_ :		List[Any]					,  lowerCAmelCase_ :		int					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		Union[str, Any]					,  lowerCAmelCase_ :		Tuple     )							->						Optional[Any]:
     __lowerCAmelCase								=	AlbertForPreTraining(config=lowerCAmelCase_     )
     model.to(lowerCAmelCase_     )
     model.eval()
     __lowerCAmelCase								=	model(
         lowerCAmelCase_					,  attention_mask=lowerCAmelCase_					,  token_type_ids=lowerCAmelCase_					,  labels=lowerCAmelCase_					,  sentence_order_label=lowerCAmelCase_					,  )
     self.parent.assertEqual(result.prediction_logits.shape					,  (self.batch_size, self.seq_length, self.vocab_size)     )
     self.parent.assertEqual(result.sop_logits.shape					,  (self.batch_size, config.num_labels)     )






  def 				lowercase		(				self :		str					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		List[Any]					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		Dict					,  lowerCAmelCase_ :		Union[str, Any]					,  lowerCAmelCase_ :		int     )							->						Any:
     __lowerCAmelCase								=	AlbertForMaskedLM(config=lowerCAmelCase_     )
     model.to(lowerCAmelCase_     )
     model.eval()
     __lowerCAmelCase								=	model(lowerCAmelCase_					,  attention_mask=lowerCAmelCase_					,  token_type_ids=lowerCAmelCase_					,  labels=lowerCAmelCase_     )
     self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.vocab_size)     )






  def 				lowercase		(				self :		str					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		Tuple					,  lowerCAmelCase_ :		int					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		int					,  lowerCAmelCase_ :		Any     )							->						Tuple:
     __lowerCAmelCase								=	AlbertForQuestionAnswering(config=lowerCAmelCase_     )
     model.to(lowerCAmelCase_     )
     model.eval()
     __lowerCAmelCase								=	model(
         lowerCAmelCase_					,  attention_mask=lowerCAmelCase_					,  token_type_ids=lowerCAmelCase_					,  start_positions=lowerCAmelCase_					,  end_positions=lowerCAmelCase_					,  )
     self.parent.assertEqual(result.start_logits.shape					,  (self.batch_size, self.seq_length)     )
     self.parent.assertEqual(result.end_logits.shape					,  (self.batch_size, self.seq_length)     )






  def 				lowercase		(				self :		Optional[Any]					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		Dict					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		Union[str, Any]					,  lowerCAmelCase_ :		List[Any]					,  lowerCAmelCase_ :		Any     )							->						Union[str, Any]:
     __lowerCAmelCase								=	self.num_labels
     __lowerCAmelCase								=	AlbertForSequenceClassification(lowerCAmelCase_     )
     model.to(lowerCAmelCase_     )
     model.eval()
     __lowerCAmelCase								=	model(lowerCAmelCase_					,  attention_mask=lowerCAmelCase_					,  token_type_ids=lowerCAmelCase_					,  labels=lowerCAmelCase_     )
     self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_labels)     )






  def 				lowercase		(				self :		Optional[int]					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		List[Any]					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		int					,  lowerCAmelCase_ :		int					,  lowerCAmelCase_ :		int     )							->						str:
     __lowerCAmelCase								=	self.num_labels
     __lowerCAmelCase								=	AlbertForTokenClassification(config=lowerCAmelCase_     )
     model.to(lowerCAmelCase_     )
     model.eval()
     __lowerCAmelCase								=	model(lowerCAmelCase_					,  attention_mask=lowerCAmelCase_					,  token_type_ids=lowerCAmelCase_					,  labels=lowerCAmelCase_     )
     self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.num_labels)     )






  def 				lowercase		(				self :		str					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		Optional[int]					,  lowerCAmelCase_ :		Optional[int]					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		Optional[int]     )							->						Tuple:
     __lowerCAmelCase								=	self.num_choices
     __lowerCAmelCase								=	AlbertForMultipleChoice(config=lowerCAmelCase_     )
     model.to(lowerCAmelCase_     )
     model.eval()
     __lowerCAmelCase								=	input_ids.unsqueeze(1     ).expand(-1					,  self.num_choices					,  -1     ).contiguous()
     __lowerCAmelCase								=	token_type_ids.unsqueeze(1     ).expand(-1					,  self.num_choices					,  -1     ).contiguous()
     __lowerCAmelCase								=	input_mask.unsqueeze(1     ).expand(-1					,  self.num_choices					,  -1     ).contiguous()
     __lowerCAmelCase								=	model(
         lowerCAmelCase_					,  attention_mask=lowerCAmelCase_					,  token_type_ids=lowerCAmelCase_					,  labels=lowerCAmelCase_					,  )
     self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_choices)     )






  def 				lowercase		(				self :		Optional[int]     )							->						str:
     __lowerCAmelCase								=	self.prepare_config_and_inputs()
     (
         (
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	
     )								=	config_and_inputs
     __lowerCAmelCase								=	{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
     return config, inputs_dict





@require_torch
class 				_UpperCAmelCase    (    _UpperCamelCase		,    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = (
      (
          AlbertModel,
          AlbertForPreTraining,
          AlbertForMaskedLM,
          AlbertForMultipleChoice,
          AlbertForSequenceClassification,
          AlbertForTokenClassification,
          AlbertForQuestionAnswering,
      )
      if is_torch_available()
      else ()
  )
  a_        = (
      {
          """feature-extraction""": AlbertModel,
          """fill-mask""": AlbertForMaskedLM,
          """question-answering""": AlbertForQuestionAnswering,
          """text-classification""": AlbertForSequenceClassification,
          """token-classification""": AlbertForTokenClassification,
          """zero-shot""": AlbertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  a_        = True






  def 				lowercase		(				self :		Optional[int]					,  lowerCAmelCase_ :		Optional[int]					,  lowerCAmelCase_ :		Optional[int]					,  lowerCAmelCase_ :		Dict=False     )							->						Dict:
     __lowerCAmelCase								=	super()._prepare_for_class(lowerCAmelCase_					,  lowerCAmelCase_					,  return_labels=lowerCAmelCase_     )

     if return_labels:
        if model_class in get_values(lowerCAmelCase_     ):
           __lowerCAmelCase								=	torch.zeros(
               (self.model_tester.batch_size, self.model_tester.seq_length)					,  dtype=torch.long					,  device=lowerCAmelCase_     )
           __lowerCAmelCase								=	torch.zeros(
               self.model_tester.batch_size					,  dtype=torch.long					,  device=lowerCAmelCase_     )
     return inputs_dict






  def 				lowercase		(				self :		Optional[int]     )							->						Any:
     __lowerCAmelCase								=	AlbertModelTester(self     )
     __lowerCAmelCase								=	ConfigTester(self					,  config_class=lowerCAmelCase_					,  hidden_size=3_7     )






  def 				lowercase		(				self :		List[str]     )							->						Tuple:
     self.config_tester.run_common_tests()






  def 				lowercase		(				self :		int     )							->						Tuple:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*lowerCAmelCase_     )






  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase_     )






  def 				lowercase		(				self :		Any     )							->						Optional[Any]:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase_     )






  def 				lowercase		(				self :		List[Any]     )							->						Optional[Any]:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase_     )






  def 				lowercase		(				self :		Optional[int]     )							->						Union[str, Any]:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase_     )






  def 				lowercase		(				self :		Any     )							->						str:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase_     )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Optional[Any]:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     for type in ["absolute", "relative_key", "relative_key_query"]:
        __lowerCAmelCase								=	type
        self.model_tester.create_and_check_model(*lowerCAmelCase_     )






  @slow
  def 				lowercase		(				self :		Dict     )							->						int:
     for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        __lowerCAmelCase								=	AlbertModel.from_pretrained(lowerCAmelCase_     )
        self.assertIsNotNone(lowerCAmelCase_     )





@require_torch
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  @slow
  def 				lowercase		(				self :		Tuple     )							->						int:
     __lowerCAmelCase								=	AlbertModel.from_pretrained('albert-base-v2'     )
     __lowerCAmelCase								=	torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]     )
     __lowerCAmelCase								=	torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]     )
     with torch.no_grad():
        __lowerCAmelCase								=	model(lowerCAmelCase_					,  attention_mask=lowerCAmelCase_     )[0]
     __lowerCAmelCase								=	torch.Size((1, 1_1, 7_6_8)     )
     self.assertEqual(output.shape					,  lowerCAmelCase_     )
     __lowerCAmelCase								=	torch.tensor(
         [[[-0.65_13, 1.50_35, -0.27_66], [-0.65_15, 1.50_46, -0.27_80], [-0.65_12, 1.50_49, -0.27_84]]]     )

     self.assertTrue(torch.allclose(output[:, 1:4, 1:4]					,  lowerCAmelCase_					,  atol=1e-4     )     )








from ..utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_scipy_available,
    is_torch_available,
    is_torchsde_available,
)


try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_pt_objects import *  # noqa F403
else:
     from .scheduling_consistency_models import CMStochasticIterativeScheduler
     from .scheduling_ddim import DDIMScheduler
     from .scheduling_ddim_inverse import DDIMInverseScheduler
     from .scheduling_ddim_parallel import DDIMParallelScheduler
     from .scheduling_ddpm import DDPMScheduler
     from .scheduling_ddpm_parallel import DDPMParallelScheduler
     from .scheduling_deis_multistep import DEISMultistepScheduler
     from .scheduling_dpmsolver_multistep import DPMSolverMultistepScheduler
     from .scheduling_dpmsolver_multistep_inverse import DPMSolverMultistepInverseScheduler
     from .scheduling_dpmsolver_singlestep import DPMSolverSinglestepScheduler
     from .scheduling_euler_ancestral_discrete import EulerAncestralDiscreteScheduler
     from .scheduling_euler_discrete import EulerDiscreteScheduler
     from .scheduling_heun_discrete import HeunDiscreteScheduler
     from .scheduling_ipndm import IPNDMScheduler
     from .scheduling_k_dpm_2_ancestral_discrete import KDPMaAncestralDiscreteScheduler
     from .scheduling_k_dpm_2_discrete import KDPMaDiscreteScheduler
     from .scheduling_karras_ve import KarrasVeScheduler
     from .scheduling_pndm import PNDMScheduler
     from .scheduling_repaint import RePaintScheduler
     from .scheduling_sde_ve import ScoreSdeVeScheduler
     from .scheduling_sde_vp import ScoreSdeVpScheduler
     from .scheduling_unclip import UnCLIPScheduler
     from .scheduling_unipc_multistep import UniPCMultistepScheduler
     from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin
     from .scheduling_vq_diffusion import VQDiffusionScheduler

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_flax_objects import *  # noqa F403
else:
     from .scheduling_ddim_flax import FlaxDDIMScheduler
     from .scheduling_ddpm_flax import FlaxDDPMScheduler
     from .scheduling_dpmsolver_multistep_flax import FlaxDPMSolverMultistepScheduler
     from .scheduling_karras_ve_flax import FlaxKarrasVeScheduler
     from .scheduling_lms_discrete_flax import FlaxLMSDiscreteScheduler
     from .scheduling_pndm_flax import FlaxPNDMScheduler
     from .scheduling_sde_ve_flax import FlaxScoreSdeVeScheduler
     from .scheduling_utils_flax import (
         FlaxKarrasDiffusionSchedulers,
         FlaxSchedulerMixin,
         FlaxSchedulerOutput,
         broadcast_to_shape_from_left,
     )


try:
     if not (is_torch_available() and is_scipy_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_torch_and_scipy_objects import *  # noqa F403
else:
     from .scheduling_lms_discrete import LMSDiscreteScheduler

try:
     if not (is_torch_available() and is_torchsde_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_torch_and_torchsde_objects import *  # noqa F403
else:
     from .scheduling_dpmsolver_sde import DPMSolverSDEScheduler

