




from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy


@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''		,					UpperCAmelCase__		,					)
class 				UpperCamelCase_    ( UpperCAmelCase__     ):

    '''simple docstring'''






    UpperCAmelCase__							  =      RobertaConfig
    UpperCAmelCase__							  =      '''roberta'''


    def __init__(			self      :	Any					, UpperCAmelCase__      :	List[str])						->Dict:





           '''simple docstring'''

           super().__init__(UpperCAmelCase__)

           A__       =    RobertaEmbeddings(UpperCAmelCase__)
           self.init_weights()


@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''		,					UpperCAmelCase__		,					)
class 				UpperCamelCase_    ( UpperCAmelCase__     ):

    '''simple docstring'''






    UpperCAmelCase__							  =      RobertaConfig
    UpperCAmelCase__							  =      '''roberta'''


    def __init__(			self      :	Optional[Any]					, UpperCAmelCase__      :	Tuple)						->Dict:





           '''simple docstring'''

           super().__init__(UpperCAmelCase__)
           A__       =    config.num_labels
           A__       =    config.num_hidden_layers

           A__       =    DeeRobertaModel(UpperCAmelCase__)
           A__       =    nn.Dropout(config.hidden_dropout_prob)
           A__       =    nn.Linear(config.hidden_size					, self.config.num_labels)





    @add_start_docstrings_to_model_forward(UpperCAmelCase__)
    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[Any]					, UpperCAmelCase__      :	Tuple=None					, UpperCAmelCase__      :	Union[str, Any]=None					, UpperCAmelCase__      :	Optional[Any]=None					, UpperCAmelCase__      :	Dict=None					, UpperCAmelCase__      :	List[Any]=None					, UpperCAmelCase__      :	Optional[Any]=None					, UpperCAmelCase__      :	int=None					, UpperCAmelCase__      :	Dict=-1					, UpperCAmelCase__      :	Optional[int]=False					, )						->Optional[int]:





           '''simple docstring'''

           A__       =    self.num_layers
           try:
                  A__       =    self.roberta(
                      UpperCAmelCase__					, attention_mask=UpperCAmelCase__					, token_type_ids=UpperCAmelCase__					, position_ids=UpperCAmelCase__					, head_mask=UpperCAmelCase__					, inputs_embeds=UpperCAmelCase__					, )

                  A__       =    outputs[1]

                  A__       =    self.dropout(UpperCAmelCase__)
                  A__       =    self.classifier(UpperCAmelCase__)
                  A__       =    (logits,) + outputs[2:]  # add hidden states and attention if they are here
           except HighwayException as e:
                  A__       =    e.message
                  A__       =    e.exit_layer
                  A__       =    outputs[0]

           if not self.training:
                  A__       =    entropy(UpperCAmelCase__)
                  A__       =    []
                  A__       =    []
           if labels is not None:
                  if self.num_labels == 1:
                         #  We are doing regression
                         A__       =    MSELoss()
                         A__       =    loss_fct(logits.view(-1)					, labels.view(-1))
                  else:
                         A__       =    CrossEntropyLoss()
                         A__       =    loss_fct(logits.view(-1					, self.num_labels)					, labels.view(-1))

                  # work with highway exits
                  A__       =    []
                  for highway_exit in outputs[-1]:
                         A__       =    highway_exit[0]
                         if not self.training:
                                highway_logits_all.append(UpperCAmelCase__)
                                highway_entropy.append(highway_exit[2])
                         if self.num_labels == 1:
                                #  We are doing regression
                                A__       =    MSELoss()
                                A__       =    loss_fct(highway_logits.view(-1)					, labels.view(-1))
                         else:
                                A__       =    CrossEntropyLoss()
                                A__       =    loss_fct(highway_logits.view(-1					, self.num_labels)					, labels.view(-1))
                         highway_losses.append(UpperCAmelCase__)

                  if train_highway:
                         A__       =    (sum(highway_losses[:-1]),) + outputs
                         # exclude the final highway, of course
                  else:
                         A__       =    (loss,) + outputs
           if not self.training:
                  A__       =    outputs + ((original_entropy, highway_entropy), exit_layer)
                  if output_layer >= 0:
                         A__       =    (
                             (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                         )  # use the highway of the last layer

           return outputs  # (loss), logits, (hidden_states), (attentions), entropy










def 				lowerCamelCase__      (    __lowerCamelCase			:     Tuple  ,		__lowerCamelCase			:     Union[str, Any]		):
  __UpperCAmelCase							:	Tuple           =  [1]
  for i in range(2  ,		__lowerCamelCase		):
    factorials.append(factorials[-1] * i		)
  assert 0 <= k < factorials[-1] * n, "k out of bounds"

  __UpperCAmelCase							:	Optional[Any]           =  []
  __UpperCAmelCase							:	str           =  list(range(__lowerCamelCase		)		)

  # Find permutation
  while factorials:
    __UpperCAmelCase							:	Any           =  factorials.pop()
    __UpperCAmelCase      , __UpperCAmelCase							:	Optional[Any]           =  divmod(__lowerCamelCase  ,		__lowerCamelCase		)
    permutation.append(elements[number]		)
    elements.remove(elements[number]		)
  permutation.append(elements[0]		)

  return permutation


if __name__ == "__main__":
       import doctest

       doctest.testmod()


