def forward(self, hidden_states):
    mixed_query_layer = self.query(hidden_states)
    mixed_key_layer = self.key(hidden_states)
    mixed_value_layer = self.value(hidden_states)
    query_layer = self.transpose_for_scores(mixed_query_layer)
    key_layer = self.transpose_for_scores(mixed_key_layer)
    value_layer = self.transpose_for_scores(mixed_value_layer)
    attention_scores = torch.matmul(query_layer, key_layer.transpose((- 1), (- 2)))
    attention_scores = (attention_scores / math.sqrt(self.attention_head_size))
    attention_probs = self.softmax(attention_scores)
    weights = (attention_probs if self.vis else None)
    attention_probs = self.attn_dropout(attention_probs)
    context_layer = torch.matmul(attention_probs, value_layer)
    context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
    new_context_layer_shape = (context_layer.size()[:(- 2)] + (self.all_head_size,))
    context_layer = context_layer.view(*new_context_layer_shape)
    attention_output = self.out(context_layer)
    attention_output = self.proj_dropout(attention_output)
    return (attention_output, weights)
def forward(self, hidden_states):
    mixed_query_layer = self.query(hidden_states)
    mixed_key_layer = self.key(hidden_states)
    mixed_value_layer = self.value(hidden_states)
    query_layer = self.transpose_for_scores(mixed_query_layer)
    key_layer = self.transpose_for_scores(mixed_key_layer)
    value_layer = self.transpose_for_scores(mixed_value_layer)
    attention_scores = torch.matmul(query_layer, key_layer.transpose((- 1), (- 2)))
    attention_scores = (attention_scores / math.sqrt(self.attention_head_size))
    attention_probs = self.softmax(attention_scores)
    weights = (attention_probs if self.vis else None)
    attention_probs = self.attn_dropout(attention_probs)
    context_layer = torch.matmul(attention_probs, value_layer)
    context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
    new_context_layer_shape = (context_layer.size()[:(- 2)] + (self.all_head_size,))
    context_layer = context_layer.view(*new_context_layer_shape)
    attention_output = self.out(context_layer)
    attention_output = self.proj_dropout(attention_output)
    return (attention_output, weights)