def forward(self, query, key=None, value=None, mask=None):
    (batch_size, query_len, dim) = query.size()
    assert (dim == self.dim), f'Dimensions do not match: {dim} query vs {self.dim} configured'
    n_heads = self.n_heads
    dim_per_head = (dim // n_heads)
    scale = math.sqrt(dim_per_head)

    def prepare_head(tensor):
        (bsz, seq_len, _) = tensor.size()
        tensor = tensor.view(batch_size, tensor.size(1), n_heads, dim_per_head)
        tensor = tensor.transpose(1, 2).contiguous().view((batch_size * n_heads), seq_len, dim_per_head)
        return tensor
    if ((key is None) and (value is None)):
        key = value = query
    elif (value is None):
        value = key
    (_, key_len, dim) = key.size()
    q = prepare_head(self.q_lin(query))
    k = prepare_head(self.k_lin(key))
    v = prepare_head(self.v_lin(value))
    dot_prod = q.bmm(k.transpose(1, 2))
    attn_mask = (mask == 0).view(batch_size, 1, (- 1), key_len).repeat(1, n_heads, 1, 1).expand(batch_size, n_heads, query_len, key_len).view((batch_size * n_heads), query_len, key_len)
    assert (attn_mask.shape == dot_prod.shape)
    dot_prod.masked_fill_(attn_mask, (- float(1e+20)))
    attn_weights = F.softmax((dot_prod / scale), dim=(- 1))
    attentioned = attn_weights.bmm(v)
    attentioned = attentioned.view(batch_size, n_heads, query_len, dim_per_head).transpose(1, 2).contiguous().view(batch_size, query_len, dim)
    out = self.out_lin(attentioned)
    return out
def forward(self, query, key=None, value=None, mask=None):
    (batch_size, query_len, dim) = query.size()
    assert (dim == self.dim), f'Dimensions do not match: {dim} query vs {self.dim} configured'
    n_heads = self.n_heads
    dim_per_head = (dim // n_heads)
    scale = math.sqrt(dim_per_head)

    def prepare_head(tensor):
        (bsz, seq_len, _) = tensor.size()
        tensor = tensor.view(batch_size, tensor.size(1), n_heads, dim_per_head)
        tensor = tensor.transpose(1, 2).contiguous().view((batch_size * n_heads), seq_len, dim_per_head)
        return tensor
    if ((key is None) and (value is None)):
        key = value = query
    elif (value is None):
        value = key
    (_, key_len, dim) = key.size()
    q = prepare_head(self.q_lin(query))
    k = prepare_head(self.k_lin(key))
    v = prepare_head(self.v_lin(value))
    dot_prod = q.bmm(k.transpose(1, 2))
    attn_mask = (mask == 0).view(batch_size, 1, (- 1), key_len).repeat(1, n_heads, 1, 1).expand(batch_size, n_heads, query_len, key_len).view((batch_size * n_heads), query_len, key_len)
    assert (attn_mask.shape == dot_prod.shape)
    dot_prod.masked_fill_(attn_mask, (- float(1e+20)))
    attn_weights = F.softmax((dot_prod / scale), dim=(- 1))
    attentioned = attn_weights.bmm(v)
    attentioned = attentioned.view(batch_size, n_heads, query_len, dim_per_head).transpose(1, 2).contiguous().view(batch_size, query_len, dim)
    out = self.out_lin(attentioned)
    return out