
from typing import Optional

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_outputs import (
    BaseModelOutputWithNoAttention,
    BaseModelOutputWithPoolingAndNoAttention,
    ImageClassifierOutputWithNoAttention,
)
from ...modeling_utils import PreTrainedModel
from ...utils import logging
from .configuration_regnet import RegNetConfig


__lowerCAmelCase			:     str           =   logging.get_logger(__name__)

# General docstring
__lowerCAmelCase			:     Union[str, Any]           =   'RegNetConfig'

# Base docstring
__lowerCAmelCase			:     Any           =   'facebook/regnet-y-040'
__lowerCAmelCase			:     Tuple           =   [1, 1088, 7, 7]

# Image classification docstring
__lowerCAmelCase			:     List[Any]           =   'facebook/regnet-y-040'
__lowerCAmelCase			:     Optional[Any]           =   'tabby, tabby cat'

__lowerCAmelCase			:     Union[str, Any]           =   [
    'facebook/regnet-y-040',
    # See all regnet models at https://huggingface.co/models?filter=regnet
]




class        UpperCAmelCase_			( nn.Module     ):


  '''simple docstring'''







  def __init__( self			: List[str]			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int = 3			,							UpperCamelCase__			: int = 1			,							UpperCamelCase__			: int = 1			,							UpperCamelCase__			: Optional[str] = "relu"			,							)			->	int:







    """simple docstring"""






    super().__init__()
    __magic_name__				     =       nn.Convad(
        UpperCamelCase__			,							UpperCamelCase__			,							kernel_size=UpperCamelCase__			,							stride=UpperCamelCase__			,							padding=kernel_size // 2			,							groups=UpperCamelCase__			,							bias=UpperCamelCase__			,							)
    __magic_name__				     =       nn.BatchNormad(UpperCamelCase__    )
    __magic_name__				     =       ACTaFN[activation] if activation is not None else nn.Identity()





  def 							_lowercase							( self			: Optional[Any]			,							UpperCamelCase__			: Tuple    )			->	Optional[int]:







    """simple docstring"""






    __magic_name__				     =       self.convolution(UpperCamelCase__    )
    __magic_name__				     =       self.normalization(UpperCamelCase__    )
    __magic_name__				     =       self.activation(UpperCamelCase__    )
    return hidden_state







class        UpperCAmelCase_			( nn.Module     ):


  '''simple docstring'''







  def __init__( self			: Optional[Any]			,							UpperCamelCase__			: RegNetConfig    )			->	Optional[Any]:







    """simple docstring"""






    super().__init__()
    __magic_name__				     =       RegNetConvLayer(
        config.num_channels			,							config.embedding_size			,							kernel_size=3			,							stride=2			,							activation=config.hidden_act    )
    __magic_name__				     =       config.num_channels





  def 							_lowercase							( self			: List[str]			,							UpperCamelCase__			: str    )			->	Union[str, Any]:







    """simple docstring"""






    __magic_name__				     =       pixel_values.shape[1]
    if num_channels != self.num_channels:
      raise ValueError(
          """Make sure that the channel dimension of the pixel values match with the one set in the configuration."""    )
    __magic_name__				     =       self.embedder(UpperCamelCase__    )
    return hidden_state







class        UpperCAmelCase_			( nn.Module     ):


  '''simple docstring'''







  def __init__( self			: Union[str, Any]			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int = 2    )			->	Any:







    """simple docstring"""






    super().__init__()
    __magic_name__				     =       nn.Convad(UpperCamelCase__			,							UpperCamelCase__			,							kernel_size=1			,							stride=UpperCamelCase__			,							bias=UpperCamelCase__    )
    __magic_name__				     =       nn.BatchNormad(UpperCamelCase__    )





  def 							_lowercase							( self			: Any			,							UpperCamelCase__			: Tensor    )			->	Tensor:







    """simple docstring"""






    __magic_name__				     =       self.convolution(UpperCamelCase__    )
    __magic_name__				     =       self.normalization(UpperCamelCase__    )
    return hidden_state







class        UpperCAmelCase_			( nn.Module     ):


  '''simple docstring'''







  def __init__( self			: int			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int    )			->	str:







    """simple docstring"""






    super().__init__()

    __magic_name__				     =       nn.AdaptiveAvgPoolad((1, 1)    )
    __magic_name__				     =       nn.Sequential(
        nn.Convad(UpperCamelCase__			,							UpperCamelCase__			,							kernel_size=1    )			,							nn.ReLU()			,							nn.Convad(UpperCamelCase__			,							UpperCamelCase__			,							kernel_size=1    )			,							nn.Sigmoid()			,							)





  def 							_lowercase							( self			: List[str]			,							UpperCamelCase__			: Optional[Any]    )			->	Union[str, Any]:







    """simple docstring"""






    __magic_name__				     =       self.pooler(UpperCamelCase__    )
    __magic_name__				     =       self.attention(UpperCamelCase__    )
    __magic_name__				     =       hidden_state * attention
    return hidden_state







class        UpperCAmelCase_			( nn.Module     ):


  '''simple docstring'''







  def __init__( self			: Dict			,							UpperCamelCase__			: RegNetConfig			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int = 1    )			->	Any:







    """simple docstring"""






    super().__init__()
    __magic_name__				     =       in_channels != out_channels or stride != 1
    __magic_name__				     =       max(1			,							out_channels // config.groups_width    )
    __magic_name__				     =       (
        RegNetShortCut(UpperCamelCase__			,							UpperCamelCase__			,							stride=UpperCamelCase__    ) if should_apply_shortcut else nn.Identity()
    )
    __magic_name__				     =       nn.Sequential(
        RegNetConvLayer(UpperCamelCase__			,							UpperCamelCase__			,							kernel_size=1			,							activation=config.hidden_act    )			,							RegNetConvLayer(UpperCamelCase__			,							UpperCamelCase__			,							stride=UpperCamelCase__			,							groups=UpperCamelCase__			,							activation=config.hidden_act    )			,							RegNetConvLayer(UpperCamelCase__			,							UpperCamelCase__			,							kernel_size=1			,							activation=UpperCamelCase__    )			,							)
    __magic_name__				     =       ACTaFN[config.hidden_act]





  def 							_lowercase							( self			: Union[str, Any]			,							UpperCamelCase__			: Dict    )			->	str:







    """simple docstring"""






    __magic_name__				     =       hidden_state
    __magic_name__				     =       self.layer(UpperCamelCase__    )
    __magic_name__				     =       self.shortcut(UpperCamelCase__    )
    hidden_state += residual
    __magic_name__				     =       self.activation(UpperCamelCase__    )
    return hidden_state







class        UpperCAmelCase_			( nn.Module     ):


  '''simple docstring'''







  def __init__( self			: Tuple			,							UpperCamelCase__			: RegNetConfig			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int = 1    )			->	str:







    """simple docstring"""






    super().__init__()
    __magic_name__				     =       in_channels != out_channels or stride != 1
    __magic_name__				     =       max(1			,							out_channels // config.groups_width    )
    __magic_name__				     =       (
        RegNetShortCut(UpperCamelCase__			,							UpperCamelCase__			,							stride=UpperCamelCase__    ) if should_apply_shortcut else nn.Identity()
    )
    __magic_name__				     =       nn.Sequential(
        RegNetConvLayer(UpperCamelCase__			,							UpperCamelCase__			,							kernel_size=1			,							activation=config.hidden_act    )			,							RegNetConvLayer(UpperCamelCase__			,							UpperCamelCase__			,							stride=UpperCamelCase__			,							groups=UpperCamelCase__			,							activation=config.hidden_act    )			,							RegNetSELayer(UpperCamelCase__			,							reduced_channels=int(round(in_channels / 4    )    )    )			,							RegNetConvLayer(UpperCamelCase__			,							UpperCamelCase__			,							kernel_size=1			,							activation=UpperCamelCase__    )			,							)
    __magic_name__				     =       ACTaFN[config.hidden_act]





  def 							_lowercase							( self			: Any			,							UpperCamelCase__			: Optional[int]    )			->	List[str]:







    """simple docstring"""






    __magic_name__				     =       hidden_state
    __magic_name__				     =       self.layer(UpperCamelCase__    )
    __magic_name__				     =       self.shortcut(UpperCamelCase__    )
    hidden_state += residual
    __magic_name__				     =       self.activation(UpperCamelCase__    )
    return hidden_state







class        UpperCAmelCase_			( nn.Module     ):


  '''simple docstring'''







  def __init__( self			: Optional[Any]			,							UpperCamelCase__			: RegNetConfig			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int = 2			,							UpperCamelCase__			: int = 2			,							)			->	List[Any]:







    """simple docstring"""






    super().__init__()

    __magic_name__				     =       RegNetXLayer if config.layer_type == """x""" else RegNetYLayer

    __magic_name__				     =       nn.Sequential(
        # downsampling is done in the first layer with stride of 2
        layer(
            UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__			,							stride=UpperCamelCase__			,							)			,							*[layer(UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__    ) for _ in range(depth - 1    )]			,							)





  def 							_lowercase							( self			: str			,							UpperCamelCase__			: Optional[int]    )			->	Union[str, Any]:







    """simple docstring"""






    __magic_name__				     =       self.layers(UpperCamelCase__    )
    return hidden_state







class        UpperCAmelCase_			( nn.Module     ):


  '''simple docstring'''







  def __init__( self			: int			,							UpperCamelCase__			: RegNetConfig    )			->	Optional[int]:







    """simple docstring"""






    super().__init__()
    __magic_name__				     =       nn.ModuleList([]    )
    # based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
    self.stages.append(
        RegNetStage(
            UpperCamelCase__			,							config.embedding_size			,							config.hidden_sizes[0]			,							stride=2 if config.downsample_in_first_stage else 1			,							depth=config.depths[0]			,							)    )
    __magic_name__				     =       zip(config.hidden_sizes			,							config.hidden_sizes[1:]    )
    for (in_channels, out_channels), depth in zip(UpperCamelCase__			,							config.depths[1:]    ):
      self.stages.append(RegNetStage(UpperCamelCase__			,							UpperCamelCase__			,							UpperCamelCase__			,							depth=UpperCamelCase__    )    )





  def 							_lowercase							( self			: int			,							UpperCamelCase__			: Tensor			,							UpperCamelCase__			: bool = False			,							UpperCamelCase__			: bool = True    )			->	BaseModelOutputWithNoAttention:







    """simple docstring"""






    __magic_name__				     =       () if output_hidden_states else None

    for stage_module in self.stages:
      if output_hidden_states:
        __magic_name__				     =       hidden_states + (hidden_state,)

      __magic_name__				     =       stage_module(UpperCamelCase__    )

    if output_hidden_states:
      __magic_name__				     =       hidden_states + (hidden_state,)

    if not return_dict:
      return tuple(v for v in [hidden_state, hidden_states] if v is not None    )

    return BaseModelOutputWithNoAttention(last_hidden_state=UpperCamelCase__			,							hidden_states=UpperCamelCase__    )







class        UpperCAmelCase_			( _A     ):


  '''simple docstring'''







  a__           =		RegNetConfig
  a__           =		"""regnet"""
  a__           =		"""pixel_values"""
  a__           =		True
  def 							_lowercase							( self			: int			,							UpperCamelCase__			: List[str]    )			->	Union[str, Any]:







    """simple docstring"""






    if isinstance(UpperCamelCase__			,							nn.Convad    ):
      nn.init.kaiming_normal_(module.weight			,							mode="""fan_out"""			,							nonlinearity="""relu"""    )
    elif isinstance(UpperCamelCase__			,							(nn.BatchNormad, nn.GroupNorm)    ):
      nn.init.constant_(module.weight			,							1    )
      nn.init.constant_(module.bias			,							0    )





  def 							_lowercase							( self			: Union[str, Any]			,							UpperCamelCase__			: Optional[int]			,							UpperCamelCase__			: Dict=False    )			->	Optional[int]:







    """simple docstring"""






    if isinstance(UpperCamelCase__			,							UpperCamelCase__    ):
      __magic_name__				     =       value







__lowerCAmelCase			:     Dict           =   R'\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it\n    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'

__lowerCAmelCase			:     List[Any]           =   R'\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`ConvNextImageProcessor.__call__`] for details.\n\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.\n'




@add_start_docstrings(
    """The bare RegNet model outputting raw features without any specific head on top."""  ,       _A  ,       )
# Copied from transformers.models.resnet.modeling_resnet.ResNetModel with RESNET->REGNET,ResNet->RegNet
class        UpperCAmelCase_			( _A     ):


  '''simple docstring'''







  def __init__( self			: Union[str, Any]			,							UpperCamelCase__			: List[str]    )			->	Dict:







    """simple docstring"""






    super().__init__(UpperCamelCase__    )
    __magic_name__				     =       config
    __magic_name__				     =       RegNetEmbeddings(UpperCamelCase__    )
    __magic_name__				     =       RegNetEncoder(UpperCamelCase__    )
    __magic_name__				     =       nn.AdaptiveAvgPoolad((1, 1)    )
    # Initialize weights and apply final processing
    self.post_init()





  @add_start_docstrings_to_model_forward(UpperCamelCase__    )
  @add_code_sample_docstrings(
      checkpoint=_CHECKPOINT_FOR_DOC			,							output_type=UpperCamelCase__			,							config_class=_CONFIG_FOR_DOC			,							modality="""vision"""			,							expected_output=_EXPECTED_OUTPUT_SHAPE			,							)
  def 							_lowercase							( self			: List[str]			,							UpperCamelCase__			: Tensor			,							UpperCamelCase__			: Optional[bool] = None			,							UpperCamelCase__			: Optional[bool] = None    )			->	BaseModelOutputWithPoolingAndNoAttention:







    """simple docstring"""






    __magic_name__				     =       (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    __magic_name__				     =       return_dict if return_dict is not None else self.config.use_return_dict

    __magic_name__				     =       self.embedder(UpperCamelCase__    )

    __magic_name__				     =       self.encoder(
        UpperCamelCase__			,							output_hidden_states=UpperCamelCase__			,							return_dict=UpperCamelCase__    )

    __magic_name__				     =       encoder_outputs[0]

    __magic_name__				     =       self.pooler(UpperCamelCase__    )

    if not return_dict:
      return (last_hidden_state, pooled_output) + encoder_outputs[1:]

    return BaseModelOutputWithPoolingAndNoAttention(
        last_hidden_state=UpperCamelCase__			,							pooler_output=UpperCamelCase__			,							hidden_states=encoder_outputs.hidden_states			,							)







@add_start_docstrings(
    """
    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    """  ,       _A  ,       )
# Copied from transformers.models.resnet.modeling_resnet.ResNetForImageClassification with RESNET->REGNET,ResNet->RegNet,resnet->regnet
class        UpperCAmelCase_			( _A     ):


  '''simple docstring'''







  def __init__( self			: Tuple			,							UpperCamelCase__			: Dict    )			->	Union[str, Any]:







    """simple docstring"""






    super().__init__(UpperCamelCase__    )
    __magic_name__				     =       config.num_labels
    __magic_name__				     =       RegNetModel(UpperCamelCase__    )
    # classification head
    __magic_name__				     =       nn.Sequential(
        nn.Flatten()			,							nn.Linear(config.hidden_sizes[-1]			,							config.num_labels    ) if config.num_labels > 0 else nn.Identity()			,							)
    # initialize weights and apply final processing
    self.post_init()





  @add_start_docstrings_to_model_forward(UpperCamelCase__    )
  @add_code_sample_docstrings(
      checkpoint=_IMAGE_CLASS_CHECKPOINT			,							output_type=UpperCamelCase__			,							config_class=_CONFIG_FOR_DOC			,							expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT			,							)
  def 							_lowercase							( self			: Optional[Any]			,							UpperCamelCase__			: Optional[torch.FloatTensor] = None			,							UpperCamelCase__			: Optional[torch.LongTensor] = None			,							UpperCamelCase__			: Optional[bool] = None			,							UpperCamelCase__			: Optional[bool] = None			,							)			->	ImageClassifierOutputWithNoAttention:







    """simple docstring"""






    __magic_name__				     =       return_dict if return_dict is not None else self.config.use_return_dict

    __magic_name__				     =       self.regnet(UpperCamelCase__			,							output_hidden_states=UpperCamelCase__			,							return_dict=UpperCamelCase__    )

    __magic_name__				     =       outputs.pooler_output if return_dict else outputs[1]

    __magic_name__				     =       self.classifier(UpperCamelCase__    )

    __magic_name__				     =       None

    if labels is not None:
      if self.config.problem_type is None:
        if self.num_labels == 1:
          __magic_name__				     =       """regression"""
        elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
          __magic_name__				     =       """single_label_classification"""
        else:
          __magic_name__				     =       """multi_label_classification"""
      if self.config.problem_type == "regression":
        __magic_name__				     =       MSELoss()
        if self.num_labels == 1:
          __magic_name__				     =       loss_fct(logits.squeeze()			,							labels.squeeze()    )
        else:
          __magic_name__				     =       loss_fct(UpperCamelCase__			,							UpperCamelCase__    )
      elif self.config.problem_type == "single_label_classification":
        __magic_name__				     =       CrossEntropyLoss()
        __magic_name__				     =       loss_fct(logits.view(-1			,							self.num_labels    )			,							labels.view(-1    )    )
      elif self.config.problem_type == "multi_label_classification":
        __magic_name__				     =       BCEWithLogitsLoss()
        __magic_name__				     =       loss_fct(UpperCamelCase__			,							UpperCamelCase__    )

    if not return_dict:
      __magic_name__				     =       (logits,) + outputs[2:]
      return (loss,) + output if loss is not None else output

    return ImageClassifierOutputWithNoAttention(loss=UpperCamelCase__			,							logits=UpperCamelCase__			,							hidden_states=outputs.hidden_states    )



import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        MODEL_FOR_PRETRAINING_MAPPING,
        AlbertForMaskedLM,
        AlbertForMultipleChoice,
        AlbertForPreTraining,
        AlbertForQuestionAnswering,
        AlbertForSequenceClassification,
        AlbertForTokenClassification,
        AlbertModel,
    )
    from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST




class        UpperCAmelCase_			:


  '''simple docstring'''







  def __init__( self			: Optional[Any]			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: int=13			,							UpperCamelCase__			: Optional[int]=7			,							UpperCamelCase__			: Any=True			,							UpperCamelCase__			: Optional[int]=True			,							UpperCamelCase__			: int=True			,							UpperCamelCase__			: Optional[Any]=True			,							UpperCamelCase__			: int=99			,							UpperCamelCase__			: Any=16			,							UpperCamelCase__			: str=36			,							UpperCamelCase__			: List[str]=6			,							UpperCamelCase__			: List[str]=6			,							UpperCamelCase__			: Union[str, Any]=6			,							UpperCamelCase__			: int=37			,							UpperCamelCase__			: Optional[int]="gelu"			,							UpperCamelCase__			: List[Any]=0.1			,							UpperCamelCase__			: Optional[int]=0.1			,							UpperCamelCase__			: int=512			,							UpperCamelCase__			: str=16			,							UpperCamelCase__			: int=2			,							UpperCamelCase__			: List[Any]=0.02			,							UpperCamelCase__			: Optional[int]=3			,							UpperCamelCase__			: Optional[Any]=4			,							UpperCamelCase__			: Dict=None			,							)			->	Any:







    """simple docstring"""






    __magic_name__				     =       parent
    __magic_name__				     =       batch_size
    __magic_name__				     =       seq_length
    __magic_name__				     =       is_training
    __magic_name__				     =       use_input_mask
    __magic_name__				     =       use_token_type_ids
    __magic_name__				     =       use_labels
    __magic_name__				     =       vocab_size
    __magic_name__				     =       embedding_size
    __magic_name__				     =       hidden_size
    __magic_name__				     =       num_hidden_layers
    __magic_name__				     =       num_hidden_groups
    __magic_name__				     =       num_attention_heads
    __magic_name__				     =       intermediate_size
    __magic_name__				     =       hidden_act
    __magic_name__				     =       hidden_dropout_prob
    __magic_name__				     =       attention_probs_dropout_prob
    __magic_name__				     =       max_position_embeddings
    __magic_name__				     =       type_vocab_size
    __magic_name__				     =       type_sequence_label_size
    __magic_name__				     =       initializer_range
    __magic_name__				     =       num_labels
    __magic_name__				     =       num_choices
    __magic_name__				     =       scope
  def 							_lowercase							( self			: Tuple    )			->	Dict:







    """simple docstring"""






    __magic_name__				     =       ids_tensor([self.batch_size, self.seq_length]			,							self.vocab_size    )

    __magic_name__				     =       None
    if self.use_input_mask:
      __magic_name__				     =       random_attention_mask([self.batch_size, self.seq_length]    )

    __magic_name__				     =       None
    if self.use_token_type_ids:
      __magic_name__				     =       ids_tensor([self.batch_size, self.seq_length]			,							self.type_vocab_size    )

    __magic_name__				     =       None
    __magic_name__				     =       None
    __magic_name__				     =       None
    if self.use_labels:
      __magic_name__				     =       ids_tensor([self.batch_size]			,							self.type_sequence_label_size    )
      __magic_name__				     =       ids_tensor([self.batch_size, self.seq_length]			,							self.num_labels    )
      __magic_name__				     =       ids_tensor([self.batch_size]			,							self.num_choices    )

    __magic_name__				     =       self.get_config()

    return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  def 							_lowercase							( self			: Any    )			->	List[Any]:







    """simple docstring"""






    return AlbertConfig(
        vocab_size=self.vocab_size			,							hidden_size=self.hidden_size			,							num_hidden_layers=self.num_hidden_layers			,							num_attention_heads=self.num_attention_heads			,							intermediate_size=self.intermediate_size			,							hidden_act=self.hidden_act			,							hidden_dropout_prob=self.hidden_dropout_prob			,							attention_probs_dropout_prob=self.attention_probs_dropout_prob			,							max_position_embeddings=self.max_position_embeddings			,							type_vocab_size=self.type_vocab_size			,							initializer_range=self.initializer_range			,							num_hidden_groups=self.num_hidden_groups			,							)
  def 							_lowercase							( self			: int			,							UpperCamelCase__			: Any			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: int			,							UpperCamelCase__			: Dict			,							UpperCamelCase__			: List[Any]			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: Optional[Any]    )			->	Tuple:







    """simple docstring"""






    __magic_name__				     =       AlbertModel(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__    )
    __magic_name__				     =       model(UpperCamelCase__			,							token_type_ids=UpperCamelCase__    )
    __magic_name__				     =       model(UpperCamelCase__    )
    self.parent.assertEqual(result.last_hidden_state.shape			,							(self.batch_size, self.seq_length, self.hidden_size)    )
    self.parent.assertEqual(result.pooler_output.shape			,							(self.batch_size, self.hidden_size)    )
  def 							_lowercase							( self			: Optional[int]			,							UpperCamelCase__			: int			,							UpperCamelCase__			: Optional[int]			,							UpperCamelCase__			: Dict			,							UpperCamelCase__			: int			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: Union[str, Any]    )			->	str:







    """simple docstring"""






    __magic_name__				     =       AlbertForPreTraining(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(
        UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__			,							sentence_order_label=UpperCamelCase__			,							)
    self.parent.assertEqual(result.prediction_logits.shape			,							(self.batch_size, self.seq_length, self.vocab_size)    )
    self.parent.assertEqual(result.sop_logits.shape			,							(self.batch_size, config.num_labels)    )
  def 							_lowercase							( self			: Union[str, Any]			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: int			,							UpperCamelCase__			: List[Any]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: Any			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: Tuple    )			->	Dict:







    """simple docstring"""






    __magic_name__				     =       AlbertForMaskedLM(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__    )
    self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.seq_length, self.vocab_size)    )
  def 							_lowercase							( self			: Union[str, Any]			,							UpperCamelCase__			: int			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: str			,							UpperCamelCase__			: str			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: Optional[int]			,							UpperCamelCase__			: Tuple    )			->	List[Any]:







    """simple docstring"""






    __magic_name__				     =       AlbertForQuestionAnswering(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(
        UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							start_positions=UpperCamelCase__			,							end_positions=UpperCamelCase__			,							)
    self.parent.assertEqual(result.start_logits.shape			,							(self.batch_size, self.seq_length)    )
    self.parent.assertEqual(result.end_logits.shape			,							(self.batch_size, self.seq_length)    )
  def 							_lowercase							( self			: Union[str, Any]			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: str			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: int    )			->	Tuple:







    """simple docstring"""






    __magic_name__				     =       self.num_labels
    __magic_name__				     =       AlbertForSequenceClassification(UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__    )
    self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.num_labels)    )
  def 							_lowercase							( self			: int			,							UpperCamelCase__			: List[Any]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: str			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: Optional[int]    )			->	int:







    """simple docstring"""






    __magic_name__				     =       self.num_labels
    __magic_name__				     =       AlbertForTokenClassification(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__    )
    self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.seq_length, self.num_labels)    )
  def 							_lowercase							( self			: Optional[int]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: Optional[Any]			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: List[Any]			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: int			,							UpperCamelCase__			: str    )			->	List[Any]:







    """simple docstring"""






    __magic_name__				     =       self.num_choices
    __magic_name__				     =       AlbertForMultipleChoice(config=UpperCamelCase__    )
    model.to(UpperCamelCase__    )
    model.eval()
    __magic_name__				     =       input_ids.unsqueeze(1    ).expand(-1			,							self.num_choices			,							-1    ).contiguous()
    __magic_name__				     =       token_type_ids.unsqueeze(1    ).expand(-1			,							self.num_choices			,							-1    ).contiguous()
    __magic_name__				     =       input_mask.unsqueeze(1    ).expand(-1			,							self.num_choices			,							-1    ).contiguous()
    __magic_name__				     =       model(
        UpperCamelCase__			,							attention_mask=UpperCamelCase__			,							token_type_ids=UpperCamelCase__			,							labels=UpperCamelCase__			,							)
    self.parent.assertEqual(result.logits.shape			,							(self.batch_size, self.num_choices)    )





  def 							_lowercase							( self			: int    )			->	Optional[int]:







    """simple docstring"""






    __magic_name__				     =       self.prepare_config_and_inputs()
    (
        (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    (
        __magic_name__
    )     ,    
    )				     =       config_and_inputs
    __magic_name__				     =       {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
    return config, inputs_dict







@require_torch
class        UpperCAmelCase_			( _A  ,       _A  ,       unittest.TestCase     ):


  '''simple docstring'''







  a__           =		(
      (
          AlbertModel,
          AlbertForPreTraining,
          AlbertForMaskedLM,
          AlbertForMultipleChoice,
          AlbertForSequenceClassification,
          AlbertForTokenClassification,
          AlbertForQuestionAnswering,
      )
      if is_torch_available()
      else ()
  )
  a__           =		(
      {
          """feature-extraction""": AlbertModel,
          """fill-mask""": AlbertForMaskedLM,
          """question-answering""": AlbertForQuestionAnswering,
          """text-classification""": AlbertForSequenceClassification,
          """token-classification""": AlbertForTokenClassification,
          """zero-shot""": AlbertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  a__           =		True
  def 							_lowercase							( self			: str			,							UpperCamelCase__			: Any			,							UpperCamelCase__			: Dict			,							UpperCamelCase__			: Union[str, Any]=False    )			->	Union[str, Any]:







    """simple docstring"""






    __magic_name__				     =       super()._prepare_for_class(UpperCamelCase__			,							UpperCamelCase__			,							return_labels=UpperCamelCase__    )

    if return_labels:
      if model_class in get_values(UpperCamelCase__    ):
        __magic_name__				     =       torch.zeros(
            (self.model_tester.batch_size, self.model_tester.seq_length)			,							dtype=torch.long			,							device=UpperCamelCase__    )
        __magic_name__				     =       torch.zeros(
            self.model_tester.batch_size			,							dtype=torch.long			,							device=UpperCamelCase__    )
    return inputs_dict
  def 							_lowercase							( self			: int    )			->	int:







    """simple docstring"""






    __magic_name__				     =       AlbertModelTester(self    )
    __magic_name__				     =       ConfigTester(self			,							config_class=UpperCamelCase__			,							hidden_size=37    )
  def 							_lowercase							( self			: Optional[int]    )			->	Optional[Any]:







    """simple docstring"""






    self.config_tester.run_common_tests()
  def 							_lowercase							( self			: Dict    )			->	Dict:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*UpperCamelCase__    )
  def 							_lowercase							( self			: int    )			->	List[str]:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_pretraining(*UpperCamelCase__    )
  def 							_lowercase							( self			: List[Any]    )			->	Any:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_masked_lm(*UpperCamelCase__    )
  def 							_lowercase							( self			: Dict    )			->	Tuple:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_multiple_choice(*UpperCamelCase__    )
  def 							_lowercase							( self			: Dict    )			->	List[Any]:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_question_answering(*UpperCamelCase__    )
  def 							_lowercase							( self			: Union[str, Any]    )			->	Any:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_sequence_classification(*UpperCamelCase__    )
  def 							_lowercase							( self			: Tuple    )			->	Optional[Any]:







    """simple docstring"""






    __magic_name__				     =       self.model_tester.prepare_config_and_inputs()
    for type in ["absolute", "relative_key", "relative_key_query"]:
      __magic_name__				     =       type
      self.model_tester.create_and_check_model(*UpperCamelCase__    )





  @slow
  def 							_lowercase							( self			: Optional[int]    )			->	Optional[int]:







    """simple docstring"""






    for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
      __magic_name__				     =       AlbertModel.from_pretrained(UpperCamelCase__    )
      self.assertIsNotNone(UpperCamelCase__    )







@require_torch
class        UpperCAmelCase_			( unittest.TestCase     ):


  '''simple docstring'''







  @slow
  def 							_lowercase							( self			: Dict    )			->	Union[str, Any]:







    """simple docstring"""






    __magic_name__				     =       AlbertModel.from_pretrained("""albert-base-v2"""    )
    __magic_name__				     =       torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]    )
    __magic_name__				     =       torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]    )
    with torch.no_grad():
      __magic_name__				     =       model(UpperCamelCase__			,							attention_mask=UpperCamelCase__    )[0]
    __magic_name__				     =       torch.Size((1, 11, 768)    )
    self.assertEqual(output.shape			,							UpperCamelCase__    )
    __magic_name__				     =       torch.tensor(
        [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]    )

    self.assertTrue(torch.allclose(output[:, 1:4, 1:4]			,							UpperCamelCase__			,							atol=1E-4    )    )

