





from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy







@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """						,				lowerCAmelCase_						,				)
class 				__lowerCAmelCase       (    lowerCAmelCase_):
      _lowercase				:					Union[str, Any]						=			RobertaConfig
      _lowercase				:					Tuple						=			"""roberta"""







      def __init__(    self		,      lowerCAmelCase__							)   ->						List[Any]:



        '''simple docstring'''




        super().__init__(_snake_case							)

        a__	:					Union[str, Any]										=RobertaEmbeddings(_snake_case							)
        self.init_weights()







@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """						,				lowerCAmelCase_						,				)
class 				__lowerCAmelCase       (    lowerCAmelCase_):
      _lowercase				:					Optional[int]						=			RobertaConfig
      _lowercase				:					List[Any]						=			"""roberta"""







      def __init__(    self		,      lowerCAmelCase__							)   ->						str:



        '''simple docstring'''




        super().__init__(_snake_case							)
        a__	:					Dict										=config.num_labels
        a__	:					Tuple										=config.num_hidden_layers

        a__	:					Optional[int]										=DeeRobertaModel(_snake_case							)
        a__	:					Any										=nn.Dropout(config.hidden_dropout_prob							)
        a__	:					str										=nn.Linear(config.hidden_size		,      self.config.num_labels							)







      @add_start_docstrings_to_model_forward(_snake_case							)
      def    _lowercase  (    self		,      lowerCAmelCase__=None		,      lowerCAmelCase__=None		,      lowerCAmelCase__=None		,      lowerCAmelCase__=None		,      lowerCAmelCase__=None		,      lowerCAmelCase__=None		,      lowerCAmelCase__=None		,      lowerCAmelCase__=-1		,      lowerCAmelCase__=False		,      )   ->						List[str]:



        '''simple docstring'''




        a__	:					Optional[Any]										=self.num_layers
        try:
          a__	:					Union[str, Any]										=self.roberta(
              _snake_case		,      attention_mask=_snake_case		,      token_type_ids=_snake_case		,      position_ids=_snake_case		,      head_mask=_snake_case		,      inputs_embeds=_snake_case		,      )

          a__	:					List[str]										=outputs[1]

          a__	:					int										=self.dropout(_snake_case							)
          a__	:					Union[str, Any]										=self.classifier(_snake_case							)
          a__	:					Tuple										=(logits,) + outputs[2:]  # add hidden states and attention if they are here
        except HighwayException as e:
          a__	:					str										=e.message
          a__	:					Any										=e.exit_layer
          a__	:					Any										=outputs[0]

        if not self.training:
          a__	:					Optional[int]										=entropy(_snake_case							)
          a__	:					Dict										=[]
          a__	:					Any										=[]
        if labels is not None:
          if self.num_labels == 1:
            #  We are doing regression
            a__	:					List[str]										=MSELoss()
            a__	:					Optional[Any]										=loss_fct(logits.view(-1							)		,      labels.view(-1							)							)
          else:
            a__	:					str										=CrossEntropyLoss()
            a__	:					List[Any]										=loss_fct(logits.view(-1		,      self.num_labels							)		,      labels.view(-1							)							)

          # work with highway exits
          a__	:					Optional[int]										=[]
          for highway_exit in outputs[-1]:
            a__	:					List[Any]										=highway_exit[0]
            if not self.training:
              highway_logits_all.append(_snake_case							)
              highway_entropy.append(highway_exit[2]							)
            if self.num_labels == 1:
              #  We are doing regression
              a__	:					str										=MSELoss()
              a__	:					List[str]										=loss_fct(highway_logits.view(-1							)		,      labels.view(-1							)							)
            else:
              a__	:					int										=CrossEntropyLoss()
              a__	:					Any										=loss_fct(highway_logits.view(-1		,      self.num_labels							)		,      labels.view(-1							)							)
            highway_losses.append(_snake_case							)

          if train_highway:
            a__	:					Optional[Any]										=(sum(highway_losses[:-1]							),) + outputs
            # exclude the final highway, of course
          else:
            a__	:					Optional[int]										=(loss,) + outputs
        if not self.training:
          a__	:					Optional[int]										=outputs + ((original_entropy, highway_entropy), exit_layer)
          if output_layer >= 0:
            a__	:					str										=(
                (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
            )  # use the highway of the last layer

        return outputs  # (loss), logits, (hidden_states), (attentions), entropy











import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin




class 					snake_case__						(       lowerCAmelCase_				,     unittest.TestCase     ):






     """simple docstring"""


     _SCREAMING_SNAKE_CASE						=	TransfoXLTokenizer
     _SCREAMING_SNAKE_CASE						=	False
     _SCREAMING_SNAKE_CASE						=	False






     def    lowercase_   (       self  :  Optional[int]					)   ->Any:
            super().setUp()

            snake_case__							:	Tuple   	=	[
                '<unk>',
                '[CLS]',
                '[SEP]',
                'want',
                'unwanted',
                'wa',
                'un',
                'running',
                ',',
                'low',
                'l',
            ]
            snake_case__							:	Any   	=	os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['vocab_file']					)
            with open(self.vocab_file, 'w', encoding='utf-8'					) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)






     def    lowercase_   (       self  :  Union[str, Any], **_snake_case  :  List[Any]					)   ->Dict:
            snake_case__							:	str   	=	True
            return TransfoXLTokenizer.from_pretrained(self.tmpdirname, **_snake_case					)






     def    lowercase_   (       self  :  Optional[Any], _snake_case  :  str					)   ->Dict:
            snake_case__							:	List[Any]   	=	'<unk> UNwanted , running'
            snake_case__							:	List[Any]   	=	'<unk> unwanted, running'
            return input_text, output_text






     def    lowercase_   (       self  :  List[Any]					)   ->Tuple:
            snake_case__							:	Dict   	=	TransfoXLTokenizer(vocab_file=self.vocab_file, lower_case=_snake_case					)

            snake_case__							:	str   	=	tokenizer.tokenize('<unk> UNwanted , running'					)
            self.assertListEqual(_snake_case, ['<unk>', 'unwanted', ',', 'running']					)

            self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case					), [0, 4, 8, 7]					)






     def    lowercase_   (       self  :  List[str]					)   ->List[Any]:
            snake_case__							:	str   	=	TransfoXLTokenizer(lower_case=_snake_case					)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '					), ['hello', '!', 'how', 'are', 'you', '?']					)






     def    lowercase_   (       self  :  Optional[int]					)   ->Optional[Any]:
            snake_case__							:	Optional[int]   	=	TransfoXLTokenizer(lower_case=_snake_case					)

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '					), ['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)






     def    lowercase_   (       self  :  Optional[int]					)   ->Union[str, Any]:
            snake_case__							:	List[Any]   	=	TransfoXLTokenizer(lower_case=_snake_case					)
            snake_case__							:	Dict   	=	'Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
            snake_case__							:	List[Any]   	=	[
                'Hello',
                '(',
                'bracket',
                ')',
                'and',
                'side',
                '@-@',
                'scrolled',
                '[',
                'and',
                ']',
                'Henry',
                '\'s',
                '$',
                '5',
                '@,@',
                '000',
                'with',
                '3',
                '@.@',
                '34',
                'm',
                '.',
                'What',
                '\'s',
                'up',
                '!',
                '?',
            ]

            self.assertListEqual(tokenizer.tokenize(_snake_case					), _snake_case					)

            self.assertEqual(tokenizer.convert_tokens_to_string(_snake_case					), _snake_case					)






     def    lowercase_   (       self  :  Dict					)   ->Any:
            snake_case__							:	Dict   	=	self.get_tokenizer()
            snake_case__							:	Optional[Any]   	=	len(_snake_case					)

            tokenizer.add_tokens(['new1', 'new2']					)
            tokenizer.move_added_token('new1', 1					)

            # Check that moved token is not copied (duplicate)
            self.assertEqual(len(_snake_case					), original_len + 2					)
            # Check that token is moved to specified id
            self.assertEqual(tokenizer.encode('new1'					), [1]					)
            self.assertEqual(tokenizer.decode([1]					), 'new1'					)






