def forward(self, xs, ys):
    if (self.attn == 'cosine'):
        l1 = self.cosine(xs, ys).unsqueeze((self.dim - 1))
    else:
        l1 = torch.bmm(xs, ys.transpose(1, 2))
        if (self.attn == 'sqrt'):
            d_k = ys.size((- 1))
            l1 = (l1 / math.sqrt(d_k))
    l2 = self.softmax(l1)
    lhs_emb = torch.bmm(l2, ys)
    lhs_emb = lhs_emb.add(xs)
    return (lhs_emb.squeeze((self.dim - 1)), l2)
def forward(self, xs, ys):
    if (self.attn == 'cosine'):
        l1 = self.cosine(xs, ys).unsqueeze((self.dim - 1))
    else:
        l1 = torch.bmm(xs, ys.transpose(1, 2))
        if (self.attn == 'sqrt'):
            d_k = ys.size((- 1))
            l1 = (l1 / math.sqrt(d_k))
    l2 = self.softmax(l1)
    lhs_emb = torch.bmm(l2, ys)
    lhs_emb = lhs_emb.add(xs)
    return (lhs_emb.squeeze((self.dim - 1)), l2)