




'''simple docstring'''




import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel


class   __UpperCamelCase       ( _lowerCamelCase				):
       lowercase	:    int      						='M-CLIP'






       def __init__(							self,		lowerCAmelCase=1_024,		lowerCAmelCase=768,		**lowerCAmelCase  ):



              """simple docstring"""







              lowerCamelCase_							=transformerDimSize
              lowerCamelCase_							=imageDimSize
              super().__init__(**_UpperCamelCase  )





class   __UpperCamelCase       ( _lowerCamelCase				):
       lowercase	:    str      						=MCLIPConfig






       def __init__(							self,		lowerCAmelCase,		*lowerCAmelCase,		**lowerCAmelCase  ):



              """simple docstring"""







              super().__init__(_UpperCamelCase,		*_UpperCamelCase,		**_UpperCamelCase  )
              lowerCamelCase_							=XLMRobertaModel(_UpperCamelCase  )
              lowerCamelCase_							=torch.nn.Linear(
                  in_features=config.transformerDimensions,		out_features=config.numDims  )






       def      lowercase__     (							self,		lowerCAmelCase,		lowerCAmelCase  ):



              """simple docstring"""







              lowerCamelCase_							=self.transformer(input_ids=_UpperCamelCase,		attention_mask=_UpperCamelCase  )[0]
              lowerCamelCase_							=(embs * attention_mask.unsqueeze(2  )).sum(dim=1  ) / attention_mask.sum(dim=1  )[:, None]
              return self.LinearTransformation(_UpperCamelCase  ), embs











from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


snake_case__					:							Optional[Any]													=     {'configuration_ibert': ['IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'IBertConfig', 'IBertOnnxConfig']}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case__					:							List[str]													=     [
           'IBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
           'IBertForMaskedLM',
           'IBertForMultipleChoice',
           'IBertForQuestionAnswering',
           'IBertForSequenceClassification',
           'IBertForTokenClassification',
           'IBertModel',
           'IBertPreTrainedModel',
       ]

if TYPE_CHECKING:
       from .configuration_ibert import IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, IBertConfig, IBertOnnxConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_ibert import (
                  IBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  IBertForMaskedLM,
                  IBertForMultipleChoice,
                  IBertForQuestionAnswering,
                  IBertForSequenceClassification,
                  IBertForTokenClassification,
                  IBertModel,
                  IBertPreTrainedModel,
              )

else:
       import sys

       snake_case__					:							Dict													=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





