
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase_		      =	{
    '''configuration_distilbert''': [
        '''DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''DistilBertConfig''',
        '''DistilBertOnnxConfig''',
    ],
    '''tokenization_distilbert''': ['''DistilBertTokenizer'''],
}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase_		      =	['''DistilBertTokenizerFast''']

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase_		      =	[
        '''DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''DistilBertForMaskedLM''',
        '''DistilBertForMultipleChoice''',
        '''DistilBertForQuestionAnswering''',
        '''DistilBertForSequenceClassification''',
        '''DistilBertForTokenClassification''',
        '''DistilBertModel''',
        '''DistilBertPreTrainedModel''',
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase_		      =	[
        '''TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
        '''TFDistilBertForMaskedLM''',
        '''TFDistilBertForMultipleChoice''',
        '''TFDistilBertForQuestionAnswering''',
        '''TFDistilBertForSequenceClassification''',
        '''TFDistilBertForTokenClassification''',
        '''TFDistilBertMainLayer''',
        '''TFDistilBertModel''',
        '''TFDistilBertPreTrainedModel''',
    ]

try:
    if not is_flax_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase_		      =	[
        '''FlaxDistilBertForMaskedLM''',
        '''FlaxDistilBertForMultipleChoice''',
        '''FlaxDistilBertForQuestionAnswering''',
        '''FlaxDistilBertForSequenceClassification''',
        '''FlaxDistilBertForTokenClassification''',
        '''FlaxDistilBertModel''',
        '''FlaxDistilBertPreTrainedModel''',
    ]


if TYPE_CHECKING:
    from .configuration_distilbert import (
        DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
        DistilBertConfig,
        DistilBertOnnxConfig,
    )
    from .tokenization_distilbert import DistilBertTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_distilbert_fast import DistilBertTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_distilbert import (
            DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
            DistilBertForMaskedLM,
            DistilBertForMultipleChoice,
            DistilBertForQuestionAnswering,
            DistilBertForSequenceClassification,
            DistilBertForTokenClassification,
            DistilBertModel,
            DistilBertPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_distilbert import (
            TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFDistilBertForMaskedLM,
            TFDistilBertForMultipleChoice,
            TFDistilBertForQuestionAnswering,
            TFDistilBertForSequenceClassification,
            TFDistilBertForTokenClassification,
            TFDistilBertMainLayer,
            TFDistilBertModel,
            TFDistilBertPreTrainedModel,
        )

    try:
        if not is_flax_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_flax_distilbert import (
            FlaxDistilBertForMaskedLM,
            FlaxDistilBertForMultipleChoice,
            FlaxDistilBertForQuestionAnswering,
            FlaxDistilBertForSequenceClassification,
            FlaxDistilBertForTokenClassification,
            FlaxDistilBertModel,
            FlaxDistilBertPreTrainedModel,
        )

else:
    import sys

    lowerCamelCase_		      =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)












from dataclasses import dataclass
from typing import Optional, Tuple

import torch
from torch import nn
from transformers import RobertaPreTrainedModel, XLMRobertaConfig, XLMRobertaModel
from transformers.utils import ModelOutput

@dataclass
class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   Optional[torch.FloatTensor]      =							None
							snake_case  :   torch.FloatTensor      =							None
							snake_case  :   Optional[Tuple[torch.FloatTensor]]      =							None
							snake_case  :   Optional[Tuple[torch.FloatTensor]]      =							None






class 							__SCREAMING_SNAKE_CASE       (  _a							):






							def __init__(    self	,			__lowerCAmelCase=1	,			__lowerCAmelCase=0	,			__lowerCAmelCase=2	,			__lowerCAmelCase=512	,			__lowerCAmelCase="cls"	,			__lowerCAmelCase=False	,			__lowerCAmelCase=True	,			**__lowerCAmelCase	,			):
														super().__init__(pad_token_id=__lowerCAmelCase	,			bos_token_id=__lowerCAmelCase	,			eos_token_id=__lowerCAmelCase	,			**__lowerCAmelCase	)
														UpperCamelCase__					  =      project_dim
														UpperCamelCase__					  =      pooler_fn
														UpperCamelCase__					  =      learn_encoder
														UpperCamelCase__					  =      use_attention_mask






class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   int      =							[r"""pooler""", r"""logit_scale"""]
							snake_case  :   Tuple      =							[r"""position_ids""", r"""predictions.decoder.bias"""]
							snake_case  :   str      =							"""roberta"""
							snake_case  :   Dict      =							RobertaSeriesConfig






							def __init__(    self	,			__lowerCAmelCase	):
														super().__init__(__lowerCAmelCase	)
														UpperCamelCase__					  =      XLMRobertaModel(__lowerCAmelCase	)
														UpperCamelCase__					  =      nn.Linear(config.hidden_size	,			config.project_dim	)
														UpperCamelCase__					  =      getattr(__lowerCAmelCase	,			"""has_pre_transformation"""	,			__lowerCAmelCase	)
														if self.has_pre_transformation:
																					UpperCamelCase__					  =      nn.Linear(config.hidden_size	,			config.project_dim	)
																					UpperCamelCase__					  =      nn.LayerNorm(config.hidden_size	,			eps=config.layer_norm_eps	)
														self.post_init()







							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			__lowerCAmelCase = None	,			):
														UpperCamelCase__					  =      return_dict if return_dict is not None else self.config.use_return_dict

														UpperCamelCase__					  =      self.base_model(
														    input_ids=__lowerCAmelCase	,			attention_mask=__lowerCAmelCase	,			token_type_ids=__lowerCAmelCase	,			position_ids=__lowerCAmelCase	,			head_mask=__lowerCAmelCase	,			inputs_embeds=__lowerCAmelCase	,			encoder_hidden_states=__lowerCAmelCase	,			encoder_attention_mask=__lowerCAmelCase	,			output_attentions=__lowerCAmelCase	,			output_hidden_states=True if self.has_pre_transformation else output_hidden_states	,			return_dict=__lowerCAmelCase	,			)

														if self.has_pre_transformation:
																					UpperCamelCase__					  =      outputs["""hidden_states"""][-2]
																					UpperCamelCase__					  =      self.pre_LN(__lowerCAmelCase	)
																					UpperCamelCase__					  =      self.transformation_pre(__lowerCAmelCase	)

																					return TransformationModelOutput(
																					    projection_state=__lowerCAmelCase	,			last_hidden_state=outputs.last_hidden_state	,			hidden_states=outputs.hidden_states	,			attentions=outputs.attentions	,			)
														else:
																					UpperCamelCase__					  =      self.transformation(outputs.last_hidden_state	)
																					return TransformationModelOutput(
																					    projection_state=__lowerCAmelCase	,			last_hidden_state=outputs.last_hidden_state	,			hidden_states=outputs.hidden_states	,			attentions=outputs.attentions	,			)




