






"""simple docstring"""
from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig


lowerCamelCase_					=      {
    """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/config.json""",
    """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/config.json""",
    """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/config.json""",
    """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/config.json""",
    """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/config.json""",
    """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/config.json""",
    """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/config.json""",
    """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/config.json""",
}







class 			UpperCamelCase_       (UpperCamelCase__     ):
	__magic_name__	      =				"""albert"""





	def __init__(      self					: Any	,	lowerCAmelCase_					: Union[str, Any]=30_000	,	lowerCAmelCase_					: Union[str, Any]=128	,	lowerCAmelCase_					: List[Any]=4_096	,	lowerCAmelCase_					: Union[str, Any]=12	,	lowerCAmelCase_					: Any=1	,	lowerCAmelCase_					: Dict=64	,	lowerCAmelCase_					: List[str]=16_384	,	lowerCAmelCase_					: int=1	,	lowerCAmelCase_					: List[str]="gelu_new"	,	lowerCAmelCase_					: Any=0	,	lowerCAmelCase_					: Any=0	,	lowerCAmelCase_					: Optional[int]=512	,	lowerCAmelCase_					: Tuple=2	,	lowerCAmelCase_					: int=0.0_2	,	lowerCAmelCase_					: Dict=1e-12	,	lowerCAmelCase_					: Any=0.1	,	lowerCAmelCase_					: Optional[int]="absolute"	,	lowerCAmelCase_					: int=0	,	lowerCAmelCase_					: int=2	,	lowerCAmelCase_					: Optional[Any]=3	,	**lowerCAmelCase_					: Union[str, Any]	,	)						->			int:
				super().__init__(pad_token_id=lowerCAmelCase__	,	bos_token_id=lowerCAmelCase__	,	eos_token_id=lowerCAmelCase__	,	**lowerCAmelCase__						)

				UpperCAmelCase_     : str			     =					vocab_size
				UpperCAmelCase_     : Optional[int]			     =					embedding_size
				UpperCAmelCase_     : Optional[Any]			     =					hidden_size
				UpperCAmelCase_     : List[str]			     =					num_hidden_layers
				UpperCAmelCase_     : Any			     =					num_hidden_groups
				UpperCAmelCase_     : int			     =					num_attention_heads
				UpperCAmelCase_     : int			     =					inner_group_num
				UpperCAmelCase_     : List[str]			     =					hidden_act
				UpperCAmelCase_     : Tuple			     =					intermediate_size
				UpperCAmelCase_     : List[Any]			     =					hidden_dropout_prob
				UpperCAmelCase_     : List[str]			     =					attention_probs_dropout_prob
				UpperCAmelCase_     : str			     =					max_position_embeddings
				UpperCAmelCase_     : str			     =					type_vocab_size
				UpperCAmelCase_     : List[str]			     =					initializer_range
				UpperCAmelCase_     : List[Any]			     =					layer_norm_eps
				UpperCAmelCase_     : Optional[Any]			     =					classifier_dropout_prob
				UpperCAmelCase_     : Tuple			     =					position_embedding_type







class 			UpperCamelCase_       (UpperCamelCase__     ):





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			Mapping[str, Mapping[int, str]]:




				if self.task == "multiple-choice":
							UpperCAmelCase_     : Any			     =					{0: "batch", 1: "choice", 2: "sequence"}
				else:
							UpperCAmelCase_     : Dict			     =					{0: "batch", 1: "sequence"}
				return OrderedDict(
				    [
				        ("input_ids", dynamic_axis),
				        ("attention_mask", dynamic_axis),
				        ("token_type_ids", dynamic_axis),
				    ]						)










import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					Tuple						=			"""M-CLIP"""







      def __init__(    self		,      lowerCAmelCase__=1_0_2_4		,      lowerCAmelCase__=7_6_8		,      **lowerCAmelCase__							)   ->						Any:



        '''simple docstring'''




        a__	:					int										=transformerDimSize
        a__	:					Dict										=imageDimSize
        super().__init__(**lowerCAmelCase__							)







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					Optional[Any]						=			MCLIPConfig







      def __init__(    self		,      lowerCAmelCase__		,      *lowerCAmelCase__		,      **lowerCAmelCase__							)   ->						List[str]:



        '''simple docstring'''




        super().__init__(lowerCAmelCase__		,      *lowerCAmelCase__		,      **lowerCAmelCase__							)
        a__	:					Tuple										=XLMRobertaModel(lowerCAmelCase__							)
        a__	:					List[str]										=torch.nn.Linear(
            in_features=config.transformerDimensions		,      out_features=config.numDims							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					Optional[Any]										=self.transformer(input_ids=lowerCAmelCase__		,      attention_mask=lowerCAmelCase__							)[0]
        a__	:					int										=(embs * attention_mask.unsqueeze(2							)).sum(dim=1							) / attention_mask.sum(dim=1							)[:, None]
        return self.LinearTransformation(lowerCAmelCase__							), embs






