def _attn(self, query, key, value, causal_mask, masked_bias, attn_dropout, attention_mask=None, head_mask=None):
    query = query.to(torch.float32)
    key = key.to(torch.float32)
    attn_weights = torch.matmul(query, key.transpose((- 1), (- 2)))
    attn_weights = torch.where(causal_mask, attn_weights, masked_bias.to(attn_weights.dtype))
    if (attention_mask is not None):
        attn_weights = (attn_weights + attention_mask)
    attn_weights = nn.Softmax(dim=(- 1))(attn_weights)
    attn_weights = attn_weights.to(value.dtype)
    attn_weights = attn_dropout(attn_weights)
    if (head_mask is not None):
        attn_weights = (attn_weights * head_mask)
    attn_output = torch.matmul(attn_weights, value)
    return (attn_output, attn_weights)
def _attn(self, query, key, value, causal_mask, masked_bias, attn_dropout, attention_mask=None, head_mask=None):
    query = query.to(torch.float32)
    key = key.to(torch.float32)
    attn_weights = torch.matmul(query, key.transpose((- 1), (- 2)))
    attn_weights = torch.where(causal_mask, attn_weights, masked_bias.to(attn_weights.dtype))
    if (attention_mask is not None):
        attn_weights = (attn_weights + attention_mask)
    attn_weights = nn.Softmax(dim=(- 1))(attn_weights)
    attn_weights = attn_weights.to(value.dtype)
    attn_weights = attn_dropout(attn_weights)
    if (head_mask is not None):
        attn_weights = (attn_weights * head_mask)
    attn_output = torch.matmul(attn_weights, value)
    return (attn_output, attn_weights)