parameters: (attention: TorchBertAttention,
                     layer_norm: Optional[TorchLayerNorm] = None,
                     is_trans_weight: bool = False)
docstring: load an attn model from huggingface bert attention model.
docstring_summary: load an attn model from huggingface bert attention model.
docstring_tokens: load an attn model from huggingface bert attention model .
function: def from_torch(attention: TorchBertAttention,
                     layer_norm: Optional[TorchLayerNorm] = None,
                     is_trans_weight: bool = False):
          """
          load an attn model from huggingface bert attention model.
          """
          ln_params = {}
          if layer_norm is not None:
              ln_params = {k: v for k, v in layer_norm.named_parameters()}
          params = {k: v for k, v in attention.named_parameters()}
          with torch.no_grad():
              if is_trans_weight:
                  # merge self.query.weight, self.query.weight and self.query.weight together as qkv.weight
                  qkv_weight = torch.cat(
                      (params['self.query.weight'], params['self.key.weight'],
                       params['self.value.weight']), 0)
                  output_weight = params['output.dense.weight']
                  k_w = params['self.key.weight']
                  v_w = params['self.value.weight']
                  q_w = params['self.query.weight']
              else:
                  # merge self.query.weight, self.query.weight and self.query.weight together as qkv.weight
                  qkv_weight = torch.clone(
                      torch.t(
                          torch.cat((params['self.query.weight'],
                                     params['self.key.weight'],
                                     params['self.value.weight']),
                                    0).contiguous()).contiguous())
                  output_weight = torch.clone(
                      torch.t(params['output.dense.weight']).contiguous())
                  k_w = torch.clone(
                      torch.t(params['self.key.weight']).contiguous())
                  v_w = torch.clone(
                      torch.t(params['self.value.weight']).contiguous())
                  q_w = torch.clone(
                      torch.t(params['self.query.weight']).contiguous())
  
              qkv_bias = torch.cat(
                  (params['self.query.bias'], params['self.key.bias'],
                   params['self.value.bias']), 0)
  
              if layer_norm is not None:
                  att = MultiHeadedAttention(
                      convert2tt_tensor(k_w),
                      convert2tt_tensor(params['self.key.bias']),
                      convert2tt_tensor(v_w),
                      convert2tt_tensor(params['self.value.bias']),
                      convert2tt_tensor(q_w),
                      convert2tt_tensor(params['self.query.bias']),
                      convert2tt_tensor(output_weight),
                      convert2tt_tensor(params['output.dense.bias']),
                      convert2tt_tensor(qkv_weight), convert2tt_tensor(qkv_bias),
                      convert2tt_tensor(params['output.LayerNorm.weight']),
                      convert2tt_tensor(params['output.LayerNorm.bias']),
                      convert2tt_tensor(ln_params['weight']),
                      convert2tt_tensor(ln_params['bias']),
                      attention.self.num_attention_heads)
              else:
                  att = MultiHeadedAttention(
                      convert2tt_tensor(k_w),
                      convert2tt_tensor(params['self.key.bias']),
                      convert2tt_tensor(v_w),
                      convert2tt_tensor(params['self.value.bias']),
                      convert2tt_tensor(q_w),
                      convert2tt_tensor(params['self.query.bias']),
                      convert2tt_tensor(output_weight),
                      convert2tt_tensor(params['output.dense.bias']),
                      convert2tt_tensor(qkv_weight), convert2tt_tensor(qkv_bias),
                      convert2tt_tensor(params['output.LayerNorm.weight']),
                      convert2tt_tensor(params['output.LayerNorm.bias']),
                      attention.self.num_attention_heads)
              return att
function_tokens: def from_torch ( attention : TorchBertAttention , layer_norm : Optional [ TorchLayerNorm ] = None , is_trans_weight : bool = False ) : ln_params = { } if layer_norm is not None : ln_params = { k : v for k , v in layer_norm . named_parameters ( ) } params = { k : v for k , v in attention . named_parameters ( ) } with torch . no_grad ( ) : if is_trans_weight : # merge self.query.weight, self.query.weight and self.query.weight together as qkv.weight qkv_weight = torch . cat ( ( params [ 'self.query.weight' ] , params [ 'self.key.weight' ] , params [ 'self.value.weight' ] ) , 0 ) output_weight = params [ 'output.dense.weight' ] k_w = params [ 'self.key.weight' ] v_w = params [ 'self.value.weight' ] q_w = params [ 'self.query.weight' ] else : # merge self.query.weight, self.query.weight and self.query.weight together as qkv.weight qkv_weight = torch . clone ( torch . t ( torch . cat ( ( params [ 'self.query.weight' ] , params [ 'self.key.weight' ] , params [ 'self.value.weight' ] ) , 0 ) . contiguous ( ) ) . contiguous ( ) ) output_weight = torch . clone ( torch . t ( params [ 'output.dense.weight' ] ) . contiguous ( ) ) k_w = torch . clone ( torch . t ( params [ 'self.key.weight' ] ) . contiguous ( ) ) v_w = torch . clone ( torch . t ( params [ 'self.value.weight' ] ) . contiguous ( ) ) q_w = torch . clone ( torch . t ( params [ 'self.query.weight' ] ) . contiguous ( ) ) qkv_bias = torch . cat ( ( params [ 'self.query.bias' ] , params [ 'self.key.bias' ] , params [ 'self.value.bias' ] ) , 0 ) if layer_norm is not None : att = MultiHeadedAttention ( convert2tt_tensor ( k_w ) , convert2tt_tensor ( params [ 'self.key.bias' ] ) , convert2tt_tensor ( v_w ) , convert2tt_tensor ( params [ 'self.value.bias' ] ) , convert2tt_tensor ( q_w ) , convert2tt_tensor ( params [ 'self.query.bias' ] ) , convert2tt_tensor ( output_weight ) , convert2tt_tensor ( params [ 'output.dense.bias' ] ) , convert2tt_tensor ( qkv_weight ) , convert2tt_tensor ( qkv_bias ) , convert2tt_tensor ( params [ 'output.LayerNorm.weight' ] ) , convert2tt_tensor ( params [ 'output.LayerNorm.bias' ] ) , convert2tt_tensor ( ln_params [ 'weight' ] ) , convert2tt_tensor ( ln_params [ 'bias' ] ) , attention . self . num_attention_heads ) else : att = MultiHeadedAttention ( convert2tt_tensor ( k_w ) , convert2tt_tensor ( params [ 'self.key.bias' ] ) , convert2tt_tensor ( v_w ) , convert2tt_tensor ( params [ 'self.value.bias' ] ) , convert2tt_tensor ( q_w ) , convert2tt_tensor ( params [ 'self.query.bias' ] ) , convert2tt_tensor ( output_weight ) , convert2tt_tensor ( params [ 'output.dense.bias' ] ) , convert2tt_tensor ( qkv_weight ) , convert2tt_tensor ( qkv_bias ) , convert2tt_tensor ( params [ 'output.LayerNorm.weight' ] ) , convert2tt_tensor ( params [ 'output.LayerNorm.bias' ] ) , attention . self . num_attention_heads ) return att