


import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)

def 	_a     ( SCREAMING_SNAKE_CASE		: int   ):







 """simple docstring"""



 UpperCamelCase__    :			Dict      				=      torch.exp(__lowerCamelCase   )
 UpperCamelCase__    :			List[Any]      				=      torch.sum(__lowerCamelCase			,       dim=1   )  # sum of exp(x_i)
 UpperCamelCase__    :			Optional[int]      				=      torch.sum(x * exp_x			,       dim=1   )  # sum of x_i * exp(x_i)
 return torch.log(__lowerCamelCase   ) - B / A

class 					__magic_name__		(   nn.Module):


     def __init__(					self							:   List[str]							,		lowerCamelCase__							:   List[Any]	)     ->					Tuple:





      '''simple docstring'''






      super().__init__()
      UpperCamelCase__    :			Optional[Any]      				=      config.output_attentions
      UpperCamelCase__    :			Optional[Any]      				=      config.output_hidden_states
      UpperCamelCase__    :			Dict      				=      nn.ModuleList([BertLayer(__lowercase	) for _ in range(config.num_hidden_layers	)]	)
      UpperCamelCase__    :			str      				=      nn.ModuleList([BertHighway(__lowercase	) for _ in range(config.num_hidden_layers	)]	)

      UpperCamelCase__    :			Optional[int]      				=      [-1 for _ in range(config.num_hidden_layers	)]


     def    UpperCAmelCase__		(					self							:   Optional[int]							,		lowerCamelCase__							:   Union[str, Any]	)     ->					Optional[int]:





      '''simple docstring'''






      if (type(__lowercase	) is float) or (type(__lowercase	) is int):
       for i in range(len(self.early_exit_entropy	)	):
        UpperCamelCase__    :			Tuple      				=      x
      else:
       UpperCamelCase__    :			Union[str, Any]      				=      x


     def    UpperCAmelCase__		(					self							:   Dict							,		lowerCamelCase__							:   Dict	)     ->					Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__    :			int      				=      pooler.state_dict()
      for highway in self.highway:
       for name, param in highway.pooler.state_dict().items():
        param.copy_(loaded_model[name]	)


     def    UpperCAmelCase__		(					self							:   str							,		lowerCamelCase__							:   Optional[int]							,		lowerCamelCase__							:   str=None							,		lowerCamelCase__							:   Any=None							,		lowerCamelCase__							:   str=None							,		lowerCamelCase__							:   List[Any]=None							,		)     ->					Dict:





      '''simple docstring'''






      UpperCamelCase__    :			Dict      				=      ()
      UpperCamelCase__    :			str      				=      ()
      UpperCamelCase__    :			Any      				=      ()
      for i, layer_module in enumerate(self.layer	):
       if self.output_hidden_states:
        UpperCamelCase__    :			List[str]      				=      all_hidden_states + (hidden_states,)

       UpperCamelCase__    :			Optional[Any]      				=      layer_module(
           __lowercase							,		__lowercase							,		head_mask[i]							,		__lowercase							,		__lowercase	)
       UpperCamelCase__    :			List[str]      				=      layer_outputs[0]

       if self.output_attentions:
        UpperCamelCase__    :			Union[str, Any]      				=      all_attentions + (layer_outputs[1],)

       UpperCamelCase__    :			int      				=      (hidden_states,)
       if self.output_hidden_states:
        UpperCamelCase__    :			List[str]      				=      current_outputs + (all_hidden_states,)
       if self.output_attentions:
        UpperCamelCase__    :			Union[str, Any]      				=      current_outputs + (all_attentions,)

       UpperCamelCase__    :			List[Any]      				=      self.highway[i](__lowercase	)
       # logits, pooled_output

       if not self.training:
        UpperCamelCase__    :			str      				=      highway_exit[0]
        UpperCamelCase__    :			str      				=      entropy(__lowercase	)
        UpperCamelCase__    :			Optional[Any]      				=      highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
        UpperCamelCase__    :			Tuple      				=      all_highway_exits + (highway_exit,)

        if highway_entropy < self.early_exit_entropy[i]:
         UpperCamelCase__    :			Optional[Any]      				=      (highway_logits,) + current_outputs[1:] + (all_highway_exits,)
         raise HighwayException(__lowercase							,		i + 1	)
       else:
        UpperCamelCase__    :			int      				=      all_highway_exits + (highway_exit,)

        # Add last layer
      if self.output_hidden_states:
       UpperCamelCase__    :			Dict      				=      all_hidden_states + (hidden_states,)

      UpperCamelCase__    :			List[Any]      				=      (hidden_states,)
      if self.output_hidden_states:
       UpperCamelCase__    :			str      				=      outputs + (all_hidden_states,)
      if self.output_attentions:
       UpperCamelCase__    :			Optional[Any]      				=      outputs + (all_attentions,)

      UpperCamelCase__    :			str      				=      outputs + (all_highway_exits,)
      return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits



@add_start_docstrings(
    "The Bert Model transformer with early exiting (DeeBERT). "    , lowercase__    , )
class 					__magic_name__		(   lowercase__):


     def __init__(					self							:   Any							,		lowerCamelCase__							:   List[Any]	)     ->					str:





      '''simple docstring'''






      super().__init__(__lowercase	)
      UpperCamelCase__    :			Dict      				=      config

      UpperCamelCase__    :			Optional[int]      				=      BertEmbeddings(__lowercase	)
      UpperCamelCase__    :			Optional[Any]      				=      DeeBertEncoder(__lowercase	)
      UpperCamelCase__    :			int      				=      BertPooler(__lowercase	)

      self.init_weights()


     def    UpperCAmelCase__		(					self							:   Dict	)     ->					Union[str, Any]:





      '''simple docstring'''






      self.encoder.init_highway_pooler(self.pooler	)


     def    UpperCAmelCase__		(					self							:   Optional[int]	)     ->					Tuple:





      '''simple docstring'''






      return self.embeddings.word_embeddings


     def    UpperCAmelCase__		(					self							:   int							,		lowerCamelCase__							:   str	)     ->					str:





      '''simple docstring'''






      UpperCamelCase__    :			List[Any]      				=      value


     def    UpperCAmelCase__		(					self							:   Optional[Any]							,		lowerCamelCase__							:   Any	)     ->					Optional[Any]:





      '''simple docstring'''






      for layer, heads in heads_to_prune.items():
       self.encoder.layer[layer].attention.prune_heads(__lowercase	)


     @add_start_docstrings_to_model_forward(__lowercase	)
     def    UpperCAmelCase__		(					self							:   Any							,		lowerCamelCase__							:   Tuple=None							,		lowerCamelCase__							:   Dict=None							,		lowerCamelCase__							:   Union[str, Any]=None							,		lowerCamelCase__							:   Optional[Any]=None							,		lowerCamelCase__							:   str=None							,		lowerCamelCase__							:   Union[str, Any]=None							,		lowerCamelCase__							:   Dict=None							,		lowerCamelCase__							:   Optional[int]=None							,		)     ->					List[Any]:





      '''simple docstring'''






      if input_ids is not None and inputs_embeds is not None:
       raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''	)
      elif input_ids is not None:
       UpperCamelCase__    :			Union[str, Any]      				=      input_ids.size()
      elif inputs_embeds is not None:
       UpperCamelCase__    :			List[str]      				=      inputs_embeds.size()[:-1]
      else:
       raise ValueError('''You have to specify either input_ids or inputs_embeds'''	)

      UpperCamelCase__    :			Union[str, Any]      				=      input_ids.device if input_ids is not None else inputs_embeds.device

      if attention_mask is None:
       UpperCamelCase__    :			Optional[int]      				=      torch.ones(__lowercase							,		device=__lowercase	)
      if encoder_attention_mask is None:
       UpperCamelCase__    :			Dict      				=      torch.ones(__lowercase							,		device=__lowercase	)
      if token_type_ids is None:
       UpperCamelCase__    :			Tuple      				=      torch.zeros(__lowercase							,		dtype=torch.long							,		device=__lowercase	)

      # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
      # ourselves in which case we just need to make it broadcastable to all heads.
      UpperCamelCase__    :			torch.Tensor      				=      self.get_extended_attention_mask(__lowercase							,		__lowercase							,		__lowercase	)

      # If a 2D ou 3D attention mask is provided for the cross-attention
      # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
      if encoder_attention_mask.dim() == 3:
       UpperCamelCase__    :			Any      				=      encoder_attention_mask[:, None, :, :]
      if encoder_attention_mask.dim() == 2:
       UpperCamelCase__    :			Optional[Any]      				=      encoder_attention_mask[:, None, None, :]

      UpperCamelCase__    :			List[str]      				=      encoder_extended_attention_mask.to(
          dtype=next(self.parameters()	).dtype	)  # fp16 compatibility
      UpperCamelCase__    :			int      				=      (1.0 - encoder_extended_attention_mask) * -10000.0

      # Prepare head mask if needed
      # 1.0 in head_mask indicate we keep the head
      # attention_probs has shape bsz x n_heads x N x N
      # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
      # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
      UpperCamelCase__    :			str      				=      self.get_head_mask(__lowercase							,		self.config.num_hidden_layers	)

      UpperCamelCase__    :			Tuple      				=      self.embeddings(
          input_ids=__lowercase							,		position_ids=__lowercase							,		token_type_ids=__lowercase							,		inputs_embeds=__lowercase	)
      UpperCamelCase__    :			Tuple      				=      self.encoder(
          __lowercase							,		attention_mask=__lowercase							,		head_mask=__lowercase							,		encoder_hidden_states=__lowercase							,		encoder_attention_mask=__lowercase							,		)
      UpperCamelCase__    :			List[Any]      				=      encoder_outputs[0]
      UpperCamelCase__    :			Optional[int]      				=      self.pooler(__lowercase	)

      UpperCamelCase__    :			Optional[int]      				=      (
          sequence_output,
          pooled_output,
      ) + encoder_outputs[
          1:
      ]  # add hidden_states and attentions if they are here
      return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits



class 					__magic_name__		(   lowercase__):


     def __init__(					self							:   str							,		lowerCamelCase__							:   int							,		lowerCamelCase__							:   List[str]	)     ->					List[str]:





      '''simple docstring'''






      UpperCamelCase__    :			Any      				=      message
      UpperCamelCase__    :			Dict      				=      exit_layer  # start from 1!



class 					__magic_name__		(   nn.Module):


     def __init__(					self							:   Any							,		lowerCamelCase__							:   int	)     ->					Union[str, Any]:





      '''simple docstring'''






      super().__init__()
      UpperCamelCase__    :			Optional[Any]      				=      BertPooler(__lowercase	)
      UpperCamelCase__    :			Union[str, Any]      				=      nn.Dropout(config.hidden_dropout_prob	)
      UpperCamelCase__    :			Optional[Any]      				=      nn.Linear(config.hidden_size							,		config.num_labels	)


     def    UpperCAmelCase__		(					self							:   Any							,		lowerCamelCase__							:   Optional[int]	)     ->					Any:





      '''simple docstring'''






      UpperCamelCase__    :			Union[str, Any]      				=      encoder_outputs[0]
      UpperCamelCase__    :			Optional[int]      				=      self.pooler(__lowercase	)
      # "return" pooler_output

      # BertModel
      UpperCamelCase__    :			Optional[int]      				=      (pooler_input, pooler_output) + encoder_outputs[1:]
      # "return" bmodel_output

      # Dropout and classification
      UpperCamelCase__    :			List[Any]      				=      bmodel_output[1]

      UpperCamelCase__    :			Tuple      				=      self.dropout(__lowercase	)
      UpperCamelCase__    :			Dict      				=      self.classifier(__lowercase	)

      return logits, pooled_output



@add_start_docstrings(
    "Bert Model (with early exiting - DeeBERT) with a classifier on top,\n    also takes care of multi-layer training. "    , lowercase__    , )
class 					__magic_name__		(   lowercase__):


     def __init__(					self							:   Union[str, Any]							,		lowerCamelCase__							:   Optional[Any]	)     ->					int:





      '''simple docstring'''






      super().__init__(__lowercase	)
      UpperCamelCase__    :			List[Any]      				=      config.num_labels
      UpperCamelCase__    :			Optional[Any]      				=      config.num_hidden_layers

      UpperCamelCase__    :			Optional[int]      				=      DeeBertModel(__lowercase	)
      UpperCamelCase__    :			List[Any]      				=      nn.Dropout(config.hidden_dropout_prob	)
      UpperCamelCase__    :			Union[str, Any]      				=      nn.Linear(config.hidden_size							,		self.config.num_labels	)

      self.init_weights()


     @add_start_docstrings_to_model_forward(__lowercase	)
     def    UpperCAmelCase__		(					self							:   Union[str, Any]							,		lowerCamelCase__							:   List[Any]=None							,		lowerCamelCase__							:   Dict=None							,		lowerCamelCase__							:   str=None							,		lowerCamelCase__							:   Optional[Any]=None							,		lowerCamelCase__							:   Any=None							,		lowerCamelCase__							:   Any=None							,		lowerCamelCase__							:   Optional[Any]=None							,		lowerCamelCase__							:   str=-1							,		lowerCamelCase__							:   List[str]=False							,		)     ->					List[Any]:





      '''simple docstring'''






      UpperCamelCase__    :			Tuple      				=      self.num_layers
      try:
       UpperCamelCase__    :			Optional[int]      				=      self.bert(
           __lowercase							,		attention_mask=__lowercase							,		token_type_ids=__lowercase							,		position_ids=__lowercase							,		head_mask=__lowercase							,		inputs_embeds=__lowercase							,		)
       # sequence_output, pooled_output, (hidden_states), (attentions), highway exits

       UpperCamelCase__    :			Optional[int]      				=      outputs[1]

       UpperCamelCase__    :			Any      				=      self.dropout(__lowercase	)
       UpperCamelCase__    :			List[str]      				=      self.classifier(__lowercase	)
       UpperCamelCase__    :			Tuple      				=      (logits,) + outputs[2:]  # add hidden states and attention if they are here
      except HighwayException as e:
       UpperCamelCase__    :			Dict      				=      e.message
       UpperCamelCase__    :			List[str]      				=      e.exit_layer
       UpperCamelCase__    :			Optional[int]      				=      outputs[0]

      if not self.training:
       UpperCamelCase__    :			Optional[int]      				=      entropy(__lowercase	)
       UpperCamelCase__    :			Optional[int]      				=      []
       UpperCamelCase__    :			Tuple      				=      []
      if labels is not None:
       if self.num_labels == 1:
        #  We are doing regression
        UpperCamelCase__    :			Union[str, Any]      				=      MSELoss()
        UpperCamelCase__    :			str      				=      loss_fct(logits.view(-1	)							,		labels.view(-1	)	)
       else:
        UpperCamelCase__    :			Optional[int]      				=      CrossEntropyLoss()
        UpperCamelCase__    :			Optional[int]      				=      loss_fct(logits.view(-1							,		self.num_labels	)							,		labels.view(-1	)	)

       # work with highway exits
       UpperCamelCase__    :			Any      				=      []
       for highway_exit in outputs[-1]:
        UpperCamelCase__    :			Optional[int]      				=      highway_exit[0]
        if not self.training:
         highway_logits_all.append(__lowercase	)
         highway_entropy.append(highway_exit[2]	)
        if self.num_labels == 1:
         #  We are doing regression
         UpperCamelCase__    :			Any      				=      MSELoss()
         UpperCamelCase__    :			List[str]      				=      loss_fct(highway_logits.view(-1	)							,		labels.view(-1	)	)
        else:
         UpperCamelCase__    :			List[Any]      				=      CrossEntropyLoss()
         UpperCamelCase__    :			Any      				=      loss_fct(highway_logits.view(-1							,		self.num_labels	)							,		labels.view(-1	)	)
        highway_losses.append(__lowercase	)

       if train_highway:
        UpperCamelCase__    :			List[str]      				=      (sum(highway_losses[:-1]	),) + outputs
        # exclude the final highway, of course
       else:
        UpperCamelCase__    :			Tuple      				=      (loss,) + outputs
      if not self.training:
       UpperCamelCase__    :			Optional[int]      				=      outputs + ((original_entropy, highway_entropy), exit_layer)
       if output_layer >= 0:
        UpperCamelCase__    :			Optional[int]      				=      (
            (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
        )  # use the highway of the last layer

      return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)










import warnings
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


a		:       List[Any]			     = logging.get_logger(__name__)

a		:       List[Any]			     = {
    "nvidia/segformer-b0-finetuned-ade-512-512": (
        "https://huggingface.co/nvidia/segformer-b0-finetuned-ade-512-512/resolve/main/config.json"
    ),
    # See all SegFormer models at https://huggingface.co/models?filter=segformer
}
class     a    ( lowercase__   ):



      """simple docstring"""


      a  : int							       =	'segformer'
      def __init__(      self					:			Dict		,      __lowercase					:			str=3		,      __lowercase					:			Dict=4		,      __lowercase					:			Any=[2, 2, 2, 2]		,      __lowercase					:			Optional[int]=[8, 4, 2, 1]		,      __lowercase					:			List[str]=[32, 64, 160, 256]		,      __lowercase					:			Union[str, Any]=[7, 3, 3, 3]		,      __lowercase					:			Optional[int]=[4, 2, 2, 2]		,      __lowercase					:			Any=[1, 2, 5, 8]		,      __lowercase					:			List[str]=[4, 4, 4, 4]		,      __lowercase					:			Any="gelu"		,      __lowercase					:			Optional[int]=0.0		,      __lowercase					:			Dict=0.0		,      __lowercase					:			Optional[Any]=0.1		,      __lowercase					:			int=0.02		,      __lowercase					:			Optional[int]=0.1		,      __lowercase					:			Optional[Any]=1e-6		,      __lowercase					:			Tuple=256		,      __lowercase					:			List[Any]=255		,      **__lowercase					:			Union[str, Any]		,      )							->      List[Any]:
        super().__init__(**__lowercase					)

        if "reshape_last_stage" in kwargs and kwargs["reshape_last_stage"] is False:
          warnings.warn(
              """Reshape_last_stage is set to False in this config. This argument is deprecated and will soon be"""
              """ removed, as the behaviour will default to that of reshape_last_stage = True."""		,      __lowercase		,      )

        __UpperCAmelCase							:	Tuple           =  num_channels
        __UpperCAmelCase							:	Any           =  num_encoder_blocks
        __UpperCAmelCase							:	List[Any]           =  depths
        __UpperCAmelCase							:	Dict           =  sr_ratios
        __UpperCAmelCase							:	int           =  hidden_sizes
        __UpperCAmelCase							:	Optional[Any]           =  patch_sizes
        __UpperCAmelCase							:	List[Any]           =  strides
        __UpperCAmelCase							:	List[str]           =  mlp_ratios
        __UpperCAmelCase							:	Tuple           =  num_attention_heads
        __UpperCAmelCase							:	List[str]           =  hidden_act
        __UpperCAmelCase							:	Tuple           =  hidden_dropout_prob
        __UpperCAmelCase							:	Dict           =  attention_probs_dropout_prob
        __UpperCAmelCase							:	Any           =  classifier_dropout_prob
        __UpperCAmelCase							:	Optional[int]           =  initializer_range
        __UpperCAmelCase							:	Optional[Any]           =  drop_path_rate
        __UpperCAmelCase							:	int           =  layer_norm_eps
        __UpperCAmelCase							:	Dict           =  decoder_hidden_size
        __UpperCAmelCase							:	Tuple           =  kwargs.get("""reshape_last_stage"""		,      __lowercase					)
        __UpperCAmelCase							:	int           =  semantic_loss_ignore_index


class     a    ( lowercase__   ):



      """simple docstring"""


      a  : Union[str, Any]							       =	version.parse('1.11'   )
      @property
      def 	UpperCAmelCase      (      self					:			Union[str, Any]					)							->      Mapping[str, Mapping[int, str]]:
        return OrderedDict(
            [
                ("""pixel_values""", {0: """batch""", 1: """num_channels""", 2: """height""", 3: """width"""}),
            ]					)
      @property
      def 	UpperCAmelCase      (      self					:			Optional[Any]					)							->      float:
        return 1e-4




      @property
      def 	UpperCAmelCase      (      self					:			str					)							->      int:
        return 12


