





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase	:	Any		     =       logging.get_logger(__name__)

UpperCAmelCase	:	Tuple		     =       {
    """facebook/s2t-wav2vec2-large-en-de""": (
        """https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/config.json"""
    ),
    # See all Speech2Text models at https://huggingface.co/models?filter=speech2text2
}







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					int						=			"""speech_to_text_2"""
      _lowercase				:					Tuple						=			["""past_key_values"""]
      _lowercase				:					Union[str, Any]						=			{"""num_attention_heads""": """decoder_attention_heads""", """hidden_size""": """d_model"""}







      def __init__(    self		,      lowerCAmelCase__=1_0_0_0_0		,      lowerCAmelCase__=6		,      lowerCAmelCase__=2_0_4_8		,      lowerCAmelCase__=4		,      lowerCAmelCase__=0.0		,      lowerCAmelCase__=True		,      lowerCAmelCase__="relu"		,      lowerCAmelCase__=2_5_6		,      lowerCAmelCase__=0.1		,      lowerCAmelCase__=0.0		,      lowerCAmelCase__=0.0		,      lowerCAmelCase__=0.02		,      lowerCAmelCase__=2		,      lowerCAmelCase__=True		,      lowerCAmelCase__=1		,      lowerCAmelCase__=0		,      lowerCAmelCase__=2		,      lowerCAmelCase__=1_0_2_4		,      **lowerCAmelCase__		,      )   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					List[str]										=vocab_size
        a__	:					List[str]										=d_model
        a__	:					List[str]										=decoder_ffn_dim
        a__	:					Union[str, Any]										=decoder_layers
        a__	:					Dict										=decoder_attention_heads
        a__	:					Union[str, Any]										=dropout
        a__	:					str										=attention_dropout
        a__	:					Tuple										=activation_dropout
        a__	:					List[str]										=activation_function
        a__	:					Dict										=init_std
        a__	:					str										=decoder_layerdrop
        a__	:					Dict										=use_cache
        a__	:					List[str]										=decoder_layers
        a__	:					List[Any]										=scale_embedding  # scale factor will be sqrt(d_model) if True
        a__	:					Dict										=max_target_positions

        super().__init__(
            pad_token_id=lowerCAmelCase__		,      bos_token_id=lowerCAmelCase__		,      eos_token_id=lowerCAmelCase__		,      decoder_start_token_id=lowerCAmelCase__		,      **lowerCAmelCase__		,      )













import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					Tuple						=			"""M-CLIP"""







      def __init__(    self		,      lowerCAmelCase__=1_0_2_4		,      lowerCAmelCase__=7_6_8		,      **lowerCAmelCase__							)   ->						Any:



        '''simple docstring'''




        a__	:					int										=transformerDimSize
        a__	:					Dict										=imageDimSize
        super().__init__(**lowerCAmelCase__							)







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					Optional[Any]						=			MCLIPConfig







      def __init__(    self		,      lowerCAmelCase__		,      *lowerCAmelCase__		,      **lowerCAmelCase__							)   ->						List[str]:



        '''simple docstring'''




        super().__init__(lowerCAmelCase__		,      *lowerCAmelCase__		,      **lowerCAmelCase__							)
        a__	:					Tuple										=XLMRobertaModel(lowerCAmelCase__							)
        a__	:					List[str]										=torch.nn.Linear(
            in_features=config.transformerDimensions		,      out_features=config.numDims							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					Optional[Any]										=self.transformer(input_ids=lowerCAmelCase__		,      attention_mask=lowerCAmelCase__							)[0]
        a__	:					int										=(embs * attention_mask.unsqueeze(2							)).sum(dim=1							) / attention_mask.sum(dim=1							)[:, None]
        return self.LinearTransformation(lowerCAmelCase__							), embs






