






import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           AlbertForMaskedLM,
           AlbertForMultipleChoice,
           AlbertForPreTraining,
           AlbertForQuestionAnswering,
           AlbertForSequenceClassification,
           AlbertForTokenClassification,
           AlbertModel,
       )
       from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class     a    :



      """simple docstring"""


      def __init__(      self					:			Any		,      __lowercase					:			List[str]		,      __lowercase					:			Any=13		,      __lowercase					:			Optional[Any]=7		,      __lowercase					:			Dict=True		,      __lowercase					:			List[Any]=True		,      __lowercase					:			Union[str, Any]=True		,      __lowercase					:			str=True		,      __lowercase					:			List[Any]=99		,      __lowercase					:			Dict=16		,      __lowercase					:			Tuple=36		,      __lowercase					:			Optional[int]=6		,      __lowercase					:			Tuple=6		,      __lowercase					:			Any=6		,      __lowercase					:			int=37		,      __lowercase					:			Any="gelu"		,      __lowercase					:			Tuple=0.1		,      __lowercase					:			List[str]=0.1		,      __lowercase					:			Optional[int]=512		,      __lowercase					:			Union[str, Any]=16		,      __lowercase					:			Dict=2		,      __lowercase					:			Union[str, Any]=0.02		,      __lowercase					:			Tuple=3		,      __lowercase					:			List[str]=4		,      __lowercase					:			Optional[int]=None		,      )							->      Any:
        __UpperCAmelCase							:	Dict           =  parent
        __UpperCAmelCase							:	Any           =  batch_size
        __UpperCAmelCase							:	Union[str, Any]           =  seq_length
        __UpperCAmelCase							:	str           =  is_training
        __UpperCAmelCase							:	int           =  use_input_mask
        __UpperCAmelCase							:	Any           =  use_token_type_ids
        __UpperCAmelCase							:	str           =  use_labels
        __UpperCAmelCase							:	Optional[Any]           =  vocab_size
        __UpperCAmelCase							:	Tuple           =  embedding_size
        __UpperCAmelCase							:	Any           =  hidden_size
        __UpperCAmelCase							:	int           =  num_hidden_layers
        __UpperCAmelCase							:	List[str]           =  num_hidden_groups
        __UpperCAmelCase							:	List[str]           =  num_attention_heads
        __UpperCAmelCase							:	Dict           =  intermediate_size
        __UpperCAmelCase							:	str           =  hidden_act
        __UpperCAmelCase							:	Optional[Any]           =  hidden_dropout_prob
        __UpperCAmelCase							:	Tuple           =  attention_probs_dropout_prob
        __UpperCAmelCase							:	Tuple           =  max_position_embeddings
        __UpperCAmelCase							:	List[Any]           =  type_vocab_size
        __UpperCAmelCase							:	Any           =  type_sequence_label_size
        __UpperCAmelCase							:	Optional[Any]           =  initializer_range
        __UpperCAmelCase							:	Tuple           =  num_labels
        __UpperCAmelCase							:	int           =  num_choices
        __UpperCAmelCase							:	Tuple           =  scope
      def 	UpperCAmelCase      (      self					:			Tuple					)							->      Dict:
        __UpperCAmelCase							:	List[Any]           =  ids_tensor([self.batch_size, self.seq_length]		,      self.vocab_size					)

        __UpperCAmelCase							:	Tuple           =  None
        if self.use_input_mask:
          __UpperCAmelCase							:	List[str]           =  random_attention_mask([self.batch_size, self.seq_length]					)

        __UpperCAmelCase							:	List[Any]           =  None
        if self.use_token_type_ids:
          __UpperCAmelCase							:	Optional[Any]           =  ids_tensor([self.batch_size, self.seq_length]		,      self.type_vocab_size					)

        __UpperCAmelCase							:	Dict           =  None
        __UpperCAmelCase							:	int           =  None
        __UpperCAmelCase							:	Optional[int]           =  None
        if self.use_labels:
          __UpperCAmelCase							:	Union[str, Any]           =  ids_tensor([self.batch_size]		,      self.type_sequence_label_size					)
          __UpperCAmelCase							:	Optional[int]           =  ids_tensor([self.batch_size, self.seq_length]		,      self.num_labels					)
          __UpperCAmelCase							:	Any           =  ids_tensor([self.batch_size]		,      self.num_choices					)

        __UpperCAmelCase							:	Any           =  self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      def 	UpperCAmelCase      (      self					:			List[Any]					)							->      List[Any]:
        return AlbertConfig(
            vocab_size=self.vocab_size		,      hidden_size=self.hidden_size		,      num_hidden_layers=self.num_hidden_layers		,      num_attention_heads=self.num_attention_heads		,      intermediate_size=self.intermediate_size		,      hidden_act=self.hidden_act		,      hidden_dropout_prob=self.hidden_dropout_prob		,      attention_probs_dropout_prob=self.attention_probs_dropout_prob		,      max_position_embeddings=self.max_position_embeddings		,      type_vocab_size=self.type_vocab_size		,      initializer_range=self.initializer_range		,      num_hidden_groups=self.num_hidden_groups		,      )
      def 	UpperCAmelCase      (      self					:			Any		,      __lowercase					:			Tuple		,      __lowercase					:			Union[str, Any]		,      __lowercase					:			str		,      __lowercase					:			Dict		,      __lowercase					:			str		,      __lowercase					:			List[str]		,      __lowercase					:			Any					)							->      Tuple:
        __UpperCAmelCase							:	Any           =  AlbertModel(config=__lowercase					)
        model.to(__lowercase					)
        model.eval()
        __UpperCAmelCase							:	Optional[int]           =  model(__lowercase		,      attention_mask=__lowercase		,      token_type_ids=__lowercase					)
        __UpperCAmelCase							:	Any           =  model(__lowercase		,      token_type_ids=__lowercase					)
        __UpperCAmelCase							:	Optional[int]           =  model(__lowercase					)
        self.parent.assertEqual(result.last_hidden_state.shape		,      (self.batch_size, self.seq_length, self.hidden_size)					)
        self.parent.assertEqual(result.pooler_output.shape		,      (self.batch_size, self.hidden_size)					)
      def 	UpperCAmelCase      (      self					:			int		,      __lowercase					:			Dict		,      __lowercase					:			Optional[int]		,      __lowercase					:			Tuple		,      __lowercase					:			str		,      __lowercase					:			Union[str, Any]		,      __lowercase					:			List[Any]		,      __lowercase					:			str					)							->      Optional[Any]:
        __UpperCAmelCase							:	List[Any]           =  AlbertForPreTraining(config=__lowercase					)
        model.to(__lowercase					)
        model.eval()
        __UpperCAmelCase							:	str           =  model(
            __lowercase		,      attention_mask=__lowercase		,      token_type_ids=__lowercase		,      labels=__lowercase		,      sentence_order_label=__lowercase		,      )
        self.parent.assertEqual(result.prediction_logits.shape		,      (self.batch_size, self.seq_length, self.vocab_size)					)
        self.parent.assertEqual(result.sop_logits.shape		,      (self.batch_size, config.num_labels)					)
      def 	UpperCAmelCase      (      self					:			Optional[int]		,      __lowercase					:			str		,      __lowercase					:			Any		,      __lowercase					:			Dict		,      __lowercase					:			Optional[Any]		,      __lowercase					:			int		,      __lowercase					:			Union[str, Any]		,      __lowercase					:			str					)							->      int:
        __UpperCAmelCase							:	Optional[int]           =  AlbertForMaskedLM(config=__lowercase					)
        model.to(__lowercase					)
        model.eval()
        __UpperCAmelCase							:	Optional[int]           =  model(__lowercase		,      attention_mask=__lowercase		,      token_type_ids=__lowercase		,      labels=__lowercase					)
        self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.vocab_size)					)
      def 	UpperCAmelCase      (      self					:			int		,      __lowercase					:			Dict		,      __lowercase					:			Optional[int]		,      __lowercase					:			Optional[int]		,      __lowercase					:			Union[str, Any]		,      __lowercase					:			Optional[Any]		,      __lowercase					:			int		,      __lowercase					:			Optional[Any]					)							->      List[str]:
        __UpperCAmelCase							:	Optional[Any]           =  AlbertForQuestionAnswering(config=__lowercase					)
        model.to(__lowercase					)
        model.eval()
        __UpperCAmelCase							:	List[str]           =  model(
            __lowercase		,      attention_mask=__lowercase		,      token_type_ids=__lowercase		,      start_positions=__lowercase		,      end_positions=__lowercase		,      )
        self.parent.assertEqual(result.start_logits.shape		,      (self.batch_size, self.seq_length)					)
        self.parent.assertEqual(result.end_logits.shape		,      (self.batch_size, self.seq_length)					)
      def 	UpperCAmelCase      (      self					:			str		,      __lowercase					:			Dict		,      __lowercase					:			Any		,      __lowercase					:			Optional[int]		,      __lowercase					:			Tuple		,      __lowercase					:			str		,      __lowercase					:			Dict		,      __lowercase					:			str					)							->      int:
        __UpperCAmelCase							:	List[str]           =  self.num_labels
        __UpperCAmelCase							:	str           =  AlbertForSequenceClassification(__lowercase					)
        model.to(__lowercase					)
        model.eval()
        __UpperCAmelCase							:	Union[str, Any]           =  model(__lowercase		,      attention_mask=__lowercase		,      token_type_ids=__lowercase		,      labels=__lowercase					)
        self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_labels)					)
      def 	UpperCAmelCase      (      self					:			Dict		,      __lowercase					:			Dict		,      __lowercase					:			Any		,      __lowercase					:			Optional[int]		,      __lowercase					:			List[Any]		,      __lowercase					:			Tuple		,      __lowercase					:			Tuple		,      __lowercase					:			Optional[int]					)							->      Optional[Any]:
        __UpperCAmelCase							:	Tuple           =  self.num_labels
        __UpperCAmelCase							:	Any           =  AlbertForTokenClassification(config=__lowercase					)
        model.to(__lowercase					)
        model.eval()
        __UpperCAmelCase							:	str           =  model(__lowercase		,      attention_mask=__lowercase		,      token_type_ids=__lowercase		,      labels=__lowercase					)
        self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.num_labels)					)
      def 	UpperCAmelCase      (      self					:			List[Any]		,      __lowercase					:			str		,      __lowercase					:			List[Any]		,      __lowercase					:			List[str]		,      __lowercase					:			Tuple		,      __lowercase					:			Any		,      __lowercase					:			List[str]		,      __lowercase					:			Dict					)							->      int:
        __UpperCAmelCase							:	str           =  self.num_choices
        __UpperCAmelCase							:	Optional[int]           =  AlbertForMultipleChoice(config=__lowercase					)
        model.to(__lowercase					)
        model.eval()
        __UpperCAmelCase							:	List[str]           =  input_ids.unsqueeze(1					).expand(-1		,      self.num_choices		,      -1					).contiguous()
        __UpperCAmelCase							:	int           =  token_type_ids.unsqueeze(1					).expand(-1		,      self.num_choices		,      -1					).contiguous()
        __UpperCAmelCase							:	Dict           =  input_mask.unsqueeze(1					).expand(-1		,      self.num_choices		,      -1					).contiguous()
        __UpperCAmelCase							:	Tuple           =  model(
            __lowercase		,      attention_mask=__lowercase		,      token_type_ids=__lowercase		,      labels=__lowercase		,      )
        self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_choices)					)




      def 	UpperCAmelCase      (      self					:			List[Any]					)							->      List[Any]:
        __UpperCAmelCase							:	List[Any]           =  self.prepare_config_and_inputs()
        (
            (
            __UpperCAmelCase
        )      , (
            __UpperCAmelCase
        )      , (
            __UpperCAmelCase
        )      , (
            __UpperCAmelCase
        )      , (
            __UpperCAmelCase
        )      , (
            __UpperCAmelCase
        )      , (
            __UpperCAmelCase
        )      , 
        )							:	Optional[int]           =  config_and_inputs
        __UpperCAmelCase							:	Tuple           =  {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
        return config, inputs_dict


@require_torch
class     a    ( lowercase__							,       lowercase__							,       unittest.TestCase   ):



      """simple docstring"""


      a  : Optional[Any]							       =	(
          (
              AlbertModel,
              AlbertForPreTraining,
              AlbertForMaskedLM,
              AlbertForMultipleChoice,
              AlbertForSequenceClassification,
              AlbertForTokenClassification,
              AlbertForQuestionAnswering,
          )
          if is_torch_available()
          else ()
      )
      a  : Tuple							       =	(
          {
              'feature-extraction': AlbertModel,
              'fill-mask': AlbertForMaskedLM,
              'question-answering': AlbertForQuestionAnswering,
              'text-classification': AlbertForSequenceClassification,
              'token-classification': AlbertForTokenClassification,
              'zero-shot': AlbertForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      a  : Union[str, Any]							       =	True
      def 	UpperCAmelCase      (      self					:			List[str]		,      __lowercase					:			List[str]		,      __lowercase					:			Optional[int]		,      __lowercase					:			Tuple=False					)							->      List[str]:
        __UpperCAmelCase							:	str           =  super()._prepare_for_class(__lowercase		,      __lowercase		,      return_labels=__lowercase					)

        if return_labels:
          if model_class in get_values(__lowercase					):
            __UpperCAmelCase							:	str           =  torch.zeros(
                (self.model_tester.batch_size, self.model_tester.seq_length)		,      dtype=torch.long		,      device=__lowercase					)
            __UpperCAmelCase							:	Tuple           =  torch.zeros(
                self.model_tester.batch_size		,      dtype=torch.long		,      device=__lowercase					)
        return inputs_dict
      def 	UpperCAmelCase      (      self					:			List[str]					)							->      int:
        __UpperCAmelCase							:	Optional[int]           =  AlbertModelTester(self					)
        __UpperCAmelCase							:	str           =  ConfigTester(self		,      config_class=__lowercase		,      hidden_size=37					)
      def 	UpperCAmelCase      (      self					:			Optional[Any]					)							->      Dict:
        self.config_tester.run_common_tests()
      def 	UpperCAmelCase      (      self					:			Dict					)							->      Dict:
        __UpperCAmelCase							:	Any           =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*__lowercase					)
      def 	UpperCAmelCase      (      self					:			Dict					)							->      Dict:
        __UpperCAmelCase							:	List[str]           =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_pretraining(*__lowercase					)
      def 	UpperCAmelCase      (      self					:			Tuple					)							->      Optional[int]:
        __UpperCAmelCase							:	int           =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*__lowercase					)
      def 	UpperCAmelCase      (      self					:			Tuple					)							->      List[Any]:
        __UpperCAmelCase							:	Optional[Any]           =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_multiple_choice(*__lowercase					)
      def 	UpperCAmelCase      (      self					:			Dict					)							->      Tuple:
        __UpperCAmelCase							:	Optional[Any]           =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*__lowercase					)
      def 	UpperCAmelCase      (      self					:			Union[str, Any]					)							->      List[str]:
        __UpperCAmelCase							:	Optional[Any]           =  self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*__lowercase					)
      def 	UpperCAmelCase      (      self					:			List[Any]					)							->      str:
        __UpperCAmelCase							:	str           =  self.model_tester.prepare_config_and_inputs()
        for type in ["absolute", "relative_key", "relative_key_query"]:
          __UpperCAmelCase							:	int           =  type
          self.model_tester.create_and_check_model(*__lowercase					)




      @slow
      def 	UpperCAmelCase      (      self					:			Union[str, Any]					)							->      Any:
        for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          __UpperCAmelCase							:	Union[str, Any]           =  AlbertModel.from_pretrained(__lowercase					)
          self.assertIsNotNone(__lowercase					)


@require_torch
class     a    ( unittest.TestCase   ):



      """simple docstring"""


      @slow
      def 	UpperCAmelCase      (      self					:			Tuple					)							->      List[str]:
        __UpperCAmelCase							:	Tuple           =  AlbertModel.from_pretrained("""albert-base-v2"""					)
        __UpperCAmelCase							:	Optional[int]           =  torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]					)
        __UpperCAmelCase							:	Dict           =  torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]					)
        with torch.no_grad():
          __UpperCAmelCase							:	str           =  model(__lowercase		,      attention_mask=__lowercase					)[0]
        __UpperCAmelCase							:	Dict           =  torch.Size((1, 11, 768)					)
        self.assertEqual(output.shape		,      __lowercase					)
        __UpperCAmelCase							:	List[Any]           =  torch.tensor(
            [[[-0.6_513, 1.5_035, -0.2_766], [-0.6_515, 1.5_046, -0.2_780], [-0.6_512, 1.5_049, -0.2_784]]]					)

        self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,      __lowercase		,      atol=1e-4					)					)










from ...configuration_utils import PretrainedConfig
from ...utils import logging


a		:       Optional[int]			     = logging.get_logger(__name__)

a		:       List[Any]			     = {
    "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/config.json",
    # See all XGLM models at https://huggingface.co/models?filter=xglm
}
class     a    ( lowercase__   ):



      """simple docstring"""


      a  : List[Any]							       =	'xglm'
      a  : str							       =	['past_key_values']

      a  : Any							       =	{
          'num_attention_heads': 'attention_heads',
          'hidden_size': 'd_model',
          'num_hidden_layers': 'num_layers',
      }
      def __init__(      self					:			Optional[int]		,      __lowercase					:			int=256008		,      __lowercase					:			Tuple=2048		,      __lowercase					:			List[Any]=1024		,      __lowercase					:			str=4096		,      __lowercase					:			Optional[Any]=24		,      __lowercase					:			Optional[int]=16		,      __lowercase					:			List[Any]="gelu"		,      __lowercase					:			str=0.1		,      __lowercase					:			Dict=0.1		,      __lowercase					:			Tuple=0.0		,      __lowercase					:			Optional[int]=0.0		,      __lowercase					:			Dict=0.02		,      __lowercase					:			Optional[int]=True		,      __lowercase					:			Any=True		,      __lowercase					:			Dict=2		,      __lowercase					:			Optional[Any]=1		,      __lowercase					:			List[Any]=0		,      __lowercase					:			Optional[Any]=2		,      **__lowercase					:			List[str]		,      )							->      Optional[int]:
        __UpperCAmelCase							:	List[str]           =  vocab_size
        __UpperCAmelCase							:	Optional[Any]           =  max_position_embeddings
        __UpperCAmelCase							:	Optional[Any]           =  d_model
        __UpperCAmelCase							:	str           =  ffn_dim
        __UpperCAmelCase							:	List[str]           =  num_layers
        __UpperCAmelCase							:	Dict           =  attention_heads
        __UpperCAmelCase							:	str           =  activation_function
        __UpperCAmelCase							:	Optional[Any]           =  dropout
        __UpperCAmelCase							:	Any           =  attention_dropout
        __UpperCAmelCase							:	int           =  activation_dropout
        __UpperCAmelCase							:	Tuple           =  layerdrop
        __UpperCAmelCase							:	Tuple           =  init_std
        __UpperCAmelCase							:	List[str]           =  scale_embedding  # scale factor will be sqrt(d_model) if True
        __UpperCAmelCase							:	Union[str, Any]           =  use_cache

        super().__init__(
            pad_token_id=__lowercase		,      bos_token_id=__lowercase		,      eos_token_id=__lowercase		,      decoder_start_token_id=__lowercase		,      **__lowercase		,      )


