




from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy







@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''			,				__magic_name__			,				)
class     __lowerCAmelCase (       __magic_name__ ):

			"""simple docstring"""







			snake_case_   		=       RobertaConfig
			snake_case_   		=       '''roberta'''


			def __init__(      self			,						lowerCamelCase__							)    ->      str:



							'''simple docstring'''
							super().__init__(lowerCamelCase__							)

							__lowerCamelCase		       =    RobertaEmbeddings(lowerCamelCase__							)
							self.init_weights()







@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''			,				__magic_name__			,				)
class     __lowerCAmelCase (       __magic_name__ ):

			"""simple docstring"""







			snake_case_   		=       RobertaConfig
			snake_case_   		=       '''roberta'''


			def __init__(      self			,						lowerCamelCase__							)    ->      Optional[Any]:



							'''simple docstring'''
							super().__init__(lowerCamelCase__							)
							__lowerCamelCase		       =    config.num_labels
							__lowerCamelCase		       =    config.num_hidden_layers

							__lowerCamelCase		       =    DeeRobertaModel(lowerCamelCase__							)
							__lowerCamelCase		       =    nn.Dropout(config.hidden_dropout_prob							)
							__lowerCamelCase		       =    nn.Linear(config.hidden_size			,						self.config.num_labels							)






			@add_start_docstrings_to_model_forward(lowerCamelCase__							)
			def       lowercase_    (      self			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=-1			,						lowerCamelCase__=False			,						)    ->      str:



							'''simple docstring'''
							__lowerCamelCase		       =    self.num_layers
							try:
											__lowerCamelCase		       =    self.roberta(
											    lowerCamelCase__			,						attention_mask=lowerCamelCase__			,						token_type_ids=lowerCamelCase__			,						position_ids=lowerCamelCase__			,						head_mask=lowerCamelCase__			,						inputs_embeds=lowerCamelCase__			,						)

											__lowerCamelCase		       =    outputs[1]

											__lowerCamelCase		       =    self.dropout(lowerCamelCase__							)
											__lowerCamelCase		       =    self.classifier(lowerCamelCase__							)
											__lowerCamelCase		       =    (logits,) + outputs[2:]  # add hidden states and attention if they are here
							except HighwayException as e:
											__lowerCamelCase		       =    e.message
											__lowerCamelCase		       =    e.exit_layer
											__lowerCamelCase		       =    outputs[0]

							if not self.training:
											__lowerCamelCase		       =    entropy(lowerCamelCase__							)
											__lowerCamelCase		       =    []
											__lowerCamelCase		       =    []
							if labels is not None:
											if self.num_labels == 1:
															#  We are doing regression
															__lowerCamelCase		       =    MSELoss()
															__lowerCamelCase		       =    loss_fct(logits.view(-1							)			,						labels.view(-1							)							)
											else:
															__lowerCamelCase		       =    CrossEntropyLoss()
															__lowerCamelCase		       =    loss_fct(logits.view(-1			,						self.num_labels							)			,						labels.view(-1							)							)

											# work with highway exits
											__lowerCamelCase		       =    []
											for highway_exit in outputs[-1]:
															__lowerCamelCase		       =    highway_exit[0]
															if not self.training:
																			highway_logits_all.append(lowerCamelCase__							)
																			highway_entropy.append(highway_exit[2]							)
															if self.num_labels == 1:
																			#  We are doing regression
																			__lowerCamelCase		       =    MSELoss()
																			__lowerCamelCase		       =    loss_fct(highway_logits.view(-1							)			,						labels.view(-1							)							)
															else:
																			__lowerCamelCase		       =    CrossEntropyLoss()
																			__lowerCamelCase		       =    loss_fct(highway_logits.view(-1			,						self.num_labels							)			,						labels.view(-1							)							)
															highway_losses.append(lowerCamelCase__							)

											if train_highway:
															__lowerCamelCase		       =    (sum(highway_losses[:-1]							),) + outputs
															# exclude the final highway, of course
											else:
															__lowerCamelCase		       =    (loss,) + outputs
							if not self.training:
											__lowerCamelCase		       =    outputs + ((original_entropy, highway_entropy), exit_layer)
											if output_layer >= 0:
															__lowerCamelCase		       =    (
															    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
															)  # use the highway of the last layer

							return outputs  # (loss), logits, (hidden_states), (attentions), entropy








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_speech_available, is_torch_available


__A        =    {
    "configuration_audio_spectrogram_transformer": [
        "AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "ASTConfig",
    ]
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__A        =    [
					    "AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "ASTForAudioClassification",
					    "ASTModel",
					    "ASTPreTrainedModel",
					]

try:
					if not is_speech_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__A        =    ["ASTFeatureExtractor"]

if TYPE_CHECKING:
					from .configuration_audio_spectrogram_transformer import (
					    AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    ASTConfig,
					)

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_audio_spectrogram_transformer import (
										    AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    ASTForAudioClassification,
										    ASTModel,
										    ASTPreTrainedModel,
										)

					try:
										if not is_speech_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .feature_extraction_audio_spectrogram_transformer import ASTFeatureExtractor


else:
					import sys

					__A        =    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


