



from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy



@add_start_docstrings(
    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). "  ,				lowerCamelCase  ,				)
class    UpperCAmelCase__(				lowerCamelCase						):

   '''simple docstring'''




   A     :	Optional[Any]												=     RobertaConfig
   A     :	int												=     "roberta"


   def __init__(    self	:      Optional[int]      ,					lowerCAmelCase	:      List[str])						->					Any:




         """simple docstring"""

         super().__init__(lowerCAmelCase)

         lowercase__  =	RobertaEmbeddings(lowerCAmelCase)
         self.init_weights()



@add_start_docstrings(
    "RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. "  ,				lowerCamelCase  ,				)
class    UpperCAmelCase__(				lowerCamelCase						):

   '''simple docstring'''




   A     :	Any												=     RobertaConfig
   A     :	List[Any]												=     "roberta"


   def __init__(    self	:      str      ,					lowerCAmelCase	:      str)						->					Tuple:




         """simple docstring"""

         super().__init__(lowerCAmelCase)
         lowercase__  =	config.num_labels
         lowercase__  =	config.num_hidden_layers

         lowercase__  =	DeeRobertaModel(lowerCAmelCase)
         lowercase__  =	nn.Dropout(config.hidden_dropout_prob)
         lowercase__  =	nn.Linear(config.hidden_size      ,					self.config.num_labels)



   @add_start_docstrings_to_model_forward(lowerCAmelCase)
   def       UpperCAmelCase		(    self	:      Dict      ,					lowerCAmelCase	:      List[str]=None      ,					lowerCAmelCase	:      Any=None      ,					lowerCAmelCase	:      Optional[Any]=None      ,					lowerCAmelCase	:      Any=None      ,					lowerCAmelCase	:      Dict=None      ,					lowerCAmelCase	:      Tuple=None      ,					lowerCAmelCase	:      Optional[int]=None      ,					lowerCAmelCase	:      Tuple=-1      ,					lowerCAmelCase	:      Dict=False      ,					)						->					List[Any]:




         """simple docstring"""

         lowercase__  =	self.num_layers
         try:
               lowercase__  =	self.roberta(
                   lowerCAmelCase      ,					attention_mask=lowerCAmelCase      ,					token_type_ids=lowerCAmelCase      ,					position_ids=lowerCAmelCase      ,					head_mask=lowerCAmelCase      ,					inputs_embeds=lowerCAmelCase      ,					)

               lowercase__  =	outputs[1]

               lowercase__  =	self.dropout(lowerCAmelCase)
               lowercase__  =	self.classifier(lowerCAmelCase)
               lowercase__  =	(logits,) + outputs[2:]  # add hidden states and attention if they are here
         except HighwayException as e:
               lowercase__  =	e.message
               lowercase__  =	e.exit_layer
               lowercase__  =	outputs[0]

         if not self.training:
               lowercase__  =	entropy(lowerCAmelCase)
               lowercase__  =	[]
               lowercase__  =	[]
         if labels is not None:
               if self.num_labels == 1:
                     #  We are doing regression
                     lowercase__  =	MSELoss()
                     lowercase__  =	loss_fct(logits.view(-1)      ,					labels.view(-1))
               else:
                     lowercase__  =	CrossEntropyLoss()
                     lowercase__  =	loss_fct(logits.view(-1      ,					self.num_labels)      ,					labels.view(-1))

               # work with highway exits
               lowercase__  =	[]
               for highway_exit in outputs[-1]:
                     lowercase__  =	highway_exit[0]
                     if not self.training:
                           highway_logits_all.append(lowerCAmelCase)
                           highway_entropy.append(highway_exit[2])
                     if self.num_labels == 1:
                           #  We are doing regression
                           lowercase__  =	MSELoss()
                           lowercase__  =	loss_fct(highway_logits.view(-1)      ,					labels.view(-1))
                     else:
                           lowercase__  =	CrossEntropyLoss()
                           lowercase__  =	loss_fct(highway_logits.view(-1      ,					self.num_labels)      ,					labels.view(-1))
                     highway_losses.append(lowerCAmelCase)

               if train_highway:
                     lowercase__  =	(sum(highway_losses[:-1]),) + outputs
                     # exclude the final highway, of course
               else:
                     lowercase__  =	(loss,) + outputs
         if not self.training:
               lowercase__  =	outputs + ((original_entropy, highway_entropy), exit_layer)
               if output_layer >= 0:
                     lowercase__  =	(
                         (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                     )  # use the highway of the last layer

         return outputs  # (loss), logits, (hidden_states), (attentions), entropy





import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin



@require_jieba
class    UpperCAmelCase__(				lowerCamelCase  ,				unittest.TestCase						):

   '''simple docstring'''




   A     :	Optional[int]												=     CpmAntTokenizer
   A     :	Optional[int]												=     False


   def       UpperCAmelCase		(    self	:      List[str])						->					str:




         """simple docstring"""

         super().setUp()

         lowercase__  =	[
             '<d>',
             '</d>',
             '<s>',
             '</s>',
             '</_>',
             '<unk>',
             '<pad>',
             '</n>',
             '我',
             '是',
             'C',
             'P',
             'M',
             'A',
             'n',
             't',
         ]
         lowercase__  =	os.path.join(self.tmpdirname      ,					VOCAB_FILES_NAMES['vocab_file'])
         with open(self.vocab_file      ,					'w'      ,					encoding='utf-8') as vocab_writer:
               vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))



   @tooslow
   def       UpperCAmelCase		(    self	:      str)						->					Optional[Any]:




         """simple docstring"""

         lowercase__  =	CpmAntTokenizer.from_pretrained('openbmb/cpm-ant-10b')
         lowercase__  =	'今天天气真好！'
         lowercase__  =	['今天', '天气', '真', '好', '！']
         lowercase__  =	tokenizer.tokenize(lowerCAmelCase)
         self.assertListEqual(lowerCAmelCase      ,					lowerCAmelCase)
         lowercase__  =	'今天天气真好！'
         lowercase__  =	[tokenizer.bos_token] + tokens

         lowercase__  =	[6, 98_02, 1_49_62, 20_82, 8_31, 2_44]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)      ,					lowerCAmelCase)

         lowercase__  =	tokenizer.decode(lowerCAmelCase)
         self.assertEqual(lowerCAmelCase      ,					lowerCAmelCase)
