

"""simple docstring"""





from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy







@add_start_docstrings(
    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). "    ,							a_    ,							)
class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					lowercase__				 =       RobertaConfig
					lowercase__				 =       "roberta"







					def __init__(       self			:				Tuple   ,lowercase_			:				Union[str, Any]		):
						super().__init__(lowercase_		)

						lowerCAmelCase__	:						Any  				=	RobertaEmbeddings(lowercase_		)
						self.init_weights()







@add_start_docstrings(
    "RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. "    ,							a_    ,							)
class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					lowercase__				 =       RobertaConfig
					lowercase__				 =       "roberta"







					def __init__(       self			:				int   ,lowercase_			:				Any		):
						super().__init__(lowercase_		)
						lowerCAmelCase__	:						Any  				=	config.num_labels
						lowerCAmelCase__	:						Any  				=	config.num_hidden_layers

						lowerCAmelCase__	:						Any  				=	DeeRobertaModel(lowercase_		)
						lowerCAmelCase__	:						Dict  				=	nn.Dropout(config.hidden_dropout_prob		)
						lowerCAmelCase__	:						Any  				=	nn.Linear(config.hidden_size   ,self.config.num_labels		)







					@add_start_docstrings_to_model_forward(lowercase_		)
					def        __lowerCAmelCase (       self			:				Optional[int]   ,lowercase_			:				int=None   ,lowercase_			:				str=None   ,lowercase_			:				Union[str, Any]=None   ,lowercase_			:				Dict=None   ,lowercase_			:				List[Any]=None   ,lowercase_			:				Dict=None   ,lowercase_			:				Dict=None   ,lowercase_			:				Any=-1   ,lowercase_			:				str=False   ,):
						lowerCAmelCase__	:						Tuple  				=	self.num_layers
						try:
							lowerCAmelCase__	:						Optional[int]  				=	self.roberta(
							    lowercase_   ,attention_mask=lowercase_   ,token_type_ids=lowercase_   ,position_ids=lowercase_   ,head_mask=lowercase_   ,inputs_embeds=lowercase_   ,)

							lowerCAmelCase__	:						Optional[Any]  				=	outputs[1]

							lowerCAmelCase__	:						Optional[int]  				=	self.dropout(lowercase_		)
							lowerCAmelCase__	:						List[str]  				=	self.classifier(lowercase_		)
							lowerCAmelCase__	:						Dict  				=	(logits,) + outputs[2:]  # add hidden states and attention if they are here
						except HighwayException as e:
							lowerCAmelCase__	:						List[Any]  				=	e.message
							lowerCAmelCase__	:						Tuple  				=	e.exit_layer
							lowerCAmelCase__	:						Dict  				=	outputs[0]

						if not self.training:
							lowerCAmelCase__	:						List[Any]  				=	entropy(lowercase_		)
							lowerCAmelCase__	:						Any  				=	[]
							lowerCAmelCase__	:						Union[str, Any]  				=	[]
						if labels is not None:
							if self.num_labels == 1:
								#  We are doing regression
								lowerCAmelCase__	:						Union[str, Any]  				=	MSELoss()
								lowerCAmelCase__	:						Tuple  				=	loss_fct(logits.view(-1		)   ,labels.view(-1		)		)
							else:
								lowerCAmelCase__	:						str  				=	CrossEntropyLoss()
								lowerCAmelCase__	:						int  				=	loss_fct(logits.view(-1   ,self.num_labels		)   ,labels.view(-1		)		)

							# work with highway exits
							lowerCAmelCase__	:						Union[str, Any]  				=	[]
							for highway_exit in outputs[-1]:
								lowerCAmelCase__	:						Union[str, Any]  				=	highway_exit[0]
								if not self.training:
									highway_logits_all.append(lowercase_		)
									highway_entropy.append(highway_exit[2]		)
								if self.num_labels == 1:
									#  We are doing regression
									lowerCAmelCase__	:						List[str]  				=	MSELoss()
									lowerCAmelCase__	:						int  				=	loss_fct(highway_logits.view(-1		)   ,labels.view(-1		)		)
								else:
									lowerCAmelCase__	:						List[Any]  				=	CrossEntropyLoss()
									lowerCAmelCase__	:						Any  				=	loss_fct(highway_logits.view(-1   ,self.num_labels		)   ,labels.view(-1		)		)
								highway_losses.append(lowercase_		)

							if train_highway:
								lowerCAmelCase__	:						Union[str, Any]  				=	(sum(highway_losses[:-1]		),) + outputs
								# exclude the final highway, of course
							else:
								lowerCAmelCase__	:						Union[str, Any]  				=	(loss,) + outputs
						if not self.training:
							lowerCAmelCase__	:						List[Any]  				=	outputs + ((original_entropy, highway_entropy), exit_layer)
							if output_layer >= 0:
								lowerCAmelCase__	:						int  				=	(
								    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
								)  # use the highway of the last layer

						return outputs  # (loss), logits, (hidden_states), (attentions), entropy







"""simple docstring"""





from ..utils import DummyObject, requires_backends







class 				SCREAMING_SNAKE_CASE					(		metaclass=a_       ):






					"""simple docstring"""



					lowercase__				 =       ["speech"]







					def __init__(       self			:				Tuple   ,*lowercase_			:				Tuple   ,**lowercase_			:				List[str]		):
						requires_backends(self   ,['''speech''']		)







class 				SCREAMING_SNAKE_CASE					(		metaclass=a_       ):






					"""simple docstring"""



					lowercase__				 =       ["speech"]







					def __init__(       self			:				Union[str, Any]   ,*lowercase_			:				List[str]   ,**lowercase_			:				Any		):
						requires_backends(self   ,['''speech''']		)




