





import os

from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE, hf_cache_home


UpperCAmelCase	:	Tuple		     =       HUGGINGFACE_HUB_CACHE


UpperCAmelCase	:	Union[str, Any]		     =       """config.json"""
UpperCAmelCase	:	Union[str, Any]		     =       """diffusion_pytorch_model.bin"""
UpperCAmelCase	:	Optional[Any]		     =       """diffusion_flax_model.msgpack"""
UpperCAmelCase	:	Optional[int]		     =       """model.onnx"""
UpperCAmelCase	:	int		     =       """diffusion_pytorch_model.safetensors"""
UpperCAmelCase	:	List[Any]		     =       """weights.pb"""
UpperCAmelCase	:	Optional[int]		     =       """https://huggingface.co"""
UpperCAmelCase	:	str		     =       default_cache_path
UpperCAmelCase	:	str		     =       """diffusers_modules"""
UpperCAmelCase	:	int		     =       os.getenv("""HF_MODULES_CACHE""", os.path.join(hf_cache_home, """modules"""))
UpperCAmelCase	:	Dict		     =       ["""fp16""", """non-ema"""]
UpperCAmelCase	:	List[str]		     =       """.self_attn"""













import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					Tuple						=			"""M-CLIP"""







      def __init__(    self		,      lowerCAmelCase__=1_0_2_4		,      lowerCAmelCase__=7_6_8		,      **lowerCAmelCase__							)   ->						Any:



        '''simple docstring'''




        a__	:					int										=transformerDimSize
        a__	:					Dict										=imageDimSize
        super().__init__(**lowerCAmelCase__							)







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					Optional[Any]						=			MCLIPConfig







      def __init__(    self		,      lowerCAmelCase__		,      *lowerCAmelCase__		,      **lowerCAmelCase__							)   ->						List[str]:



        '''simple docstring'''




        super().__init__(lowerCAmelCase__		,      *lowerCAmelCase__		,      **lowerCAmelCase__							)
        a__	:					Tuple										=XLMRobertaModel(lowerCAmelCase__							)
        a__	:					List[str]										=torch.nn.Linear(
            in_features=config.transformerDimensions		,      out_features=config.numDims							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					Optional[Any]										=self.transformer(input_ids=lowerCAmelCase__		,      attention_mask=lowerCAmelCase__							)[0]
        a__	:					int										=(embs * attention_mask.unsqueeze(2							)).sum(dim=1							) / attention_mask.sum(dim=1							)[:, None]
        return self.LinearTransformation(lowerCAmelCase__							), embs






