




from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy
@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''       ,   a_       ,   )
class 		UpperCAmelCase   (				a_			):







						'''simple docstring'''





						lowerCAmelCase_     		=   RobertaConfig
						lowerCAmelCase_     		=   '''roberta'''






						def __init__(						self							:     Tuple       ,	__lowercase							:     Union[str, Any]						):






							"""simple docstring"""






							super().__init__(lowercase_						)

							snake_case_		    =     RobertaEmbeddings(lowercase_						)
							self.init_weights()







@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. '''       ,   a_       ,   )
class 		UpperCAmelCase   (				a_			):







						'''simple docstring'''





						lowerCAmelCase_     		=   RobertaConfig
						lowerCAmelCase_     		=   '''roberta'''






						def __init__(						self							:     int       ,	__lowercase							:     Any						):






							"""simple docstring"""






							super().__init__(lowercase_						)
							snake_case_		    =     config.num_labels
							snake_case_		    =     config.num_hidden_layers

							snake_case_		    =     DeeRobertaModel(lowercase_						)
							snake_case_		    =     nn.Dropout(config.hidden_dropout_prob						)
							snake_case_		    =     nn.Linear(config.hidden_size       ,	self.config.num_labels						)






						@add_start_docstrings_to_model_forward(lowercase_						)
						def        snake_case__						(						self							:     Optional[int]       ,	__lowercase							:     int=None       ,	__lowercase							:     str=None       ,	__lowercase							:     Union[str, Any]=None       ,	__lowercase							:     Dict=None       ,	__lowercase							:     List[Any]=None       ,	__lowercase							:     Dict=None       ,	__lowercase							:     Dict=None       ,	__lowercase							:     Any=-1       ,	__lowercase							:     str=False       ,	):






							"""simple docstring"""






							snake_case_		    =     self.num_layers
							try:
								snake_case_		    =     self.roberta(
								    lowercase_       ,	attention_mask=lowercase_       ,	token_type_ids=lowercase_       ,	position_ids=lowercase_       ,	head_mask=lowercase_       ,	inputs_embeds=lowercase_       ,	)

								snake_case_		    =     outputs[1]

								snake_case_		    =     self.dropout(lowercase_						)
								snake_case_		    =     self.classifier(lowercase_						)
								snake_case_		    =     (logits,) + outputs[2:]  # add hidden states and attention if they are here
							except HighwayException as e:
								snake_case_		    =     e.message
								snake_case_		    =     e.exit_layer
								snake_case_		    =     outputs[0]

							if not self.training:
								snake_case_		    =     entropy(lowercase_						)
								snake_case_		    =     []
								snake_case_		    =     []
							if labels is not None:
								if self.num_labels == 1:
									#  We are doing regression
									snake_case_		    =     MSELoss()
									snake_case_		    =     loss_fct(logits.view(-1						)       ,	labels.view(-1						)						)
								else:
									snake_case_		    =     CrossEntropyLoss()
									snake_case_		    =     loss_fct(logits.view(-1       ,	self.num_labels						)       ,	labels.view(-1						)						)

								# work with highway exits
								snake_case_		    =     []
								for highway_exit in outputs[-1]:
									snake_case_		    =     highway_exit[0]
									if not self.training:
										highway_logits_all.append(lowercase_						)
										highway_entropy.append(highway_exit[2]						)
									if self.num_labels == 1:
										#  We are doing regression
										snake_case_		    =     MSELoss()
										snake_case_		    =     loss_fct(highway_logits.view(-1						)       ,	labels.view(-1						)						)
									else:
										snake_case_		    =     CrossEntropyLoss()
										snake_case_		    =     loss_fct(highway_logits.view(-1       ,	self.num_labels						)       ,	labels.view(-1						)						)
									highway_losses.append(lowercase_						)

								if train_highway:
									snake_case_		    =     (sum(highway_losses[:-1]						),) + outputs
									# exclude the final highway, of course
								else:
									snake_case_		    =     (loss,) + outputs
							if not self.training:
								snake_case_		    =     outputs + ((original_entropy, highway_entropy), exit_layer)
								if output_layer >= 0:
									snake_case_		    =     (
									    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
									)  # use the highway of the last layer

							return outputs  # (loss), logits, (hidden_states), (attentions), entropy



"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


__UpperCamelCase  :      Optional[Any]							      =		{
    '''configuration_rag''': ['''RagConfig'''],
    '''retrieval_rag''': ['''RagRetriever'''],
    '''tokenization_rag''': ['''RagTokenizer'''],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase  :      Tuple							      =		[
							    '''RagModel''',
							    '''RagPreTrainedModel''',
							    '''RagSequenceForGeneration''',
							    '''RagTokenForGeneration''',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase  :      Union[str, Any]							      =		[
							    '''TFRagModel''',
							    '''TFRagPreTrainedModel''',
							    '''TFRagSequenceForGeneration''',
							    '''TFRagTokenForGeneration''',
							]


if TYPE_CHECKING:
							from .configuration_rag import RagConfig
							from .retrieval_rag import RagRetriever
							from .tokenization_rag import RagTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_rag import RagModel, RagPreTrainedModel, RagSequenceForGeneration, RagTokenForGeneration

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_rag import (
														    TFRagModel,
														    TFRagPreTrainedModel,
														    TFRagSequenceForGeneration,
														    TFRagTokenForGeneration,
														)

else:
							import sys

							__UpperCamelCase  :      Dict							      =		_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




