def test_multihead_attention(self):
    x = torch.randn(16, 24, 32)
    layer = DPMultiheadAttention(32, 1)
    self._check_one_layer(layer, x, x, x, batch_first=False)
    layer = DPMultiheadAttention(32, 1, bias=True, add_bias_kv=True, dropout=0.05)
    self._check_one_layer(layer, x, x, x, batch_first=False)
    layer = DPMultiheadAttention(32, 1, bias=True, add_bias_kv=True)
    self._check_one_layer(layer, x, x, x, batch_first=False)
    layer = DPMultiheadAttention(32, 1, bias=True, add_bias_kv=True, add_zero_attn=True)
    self._check_one_layer(layer, x, x, x, batch_first=False)
    q = torch.randn(16, 24, 32)
    k = torch.randn(20, 24, 28)
    v = torch.randn(20, 24, 28)
    layer = DPMultiheadAttention(32, 1, bias=True, add_bias_kv=True, add_zero_attn=True, kdim=28, vdim=28)
    self._check_one_layer(layer, q, k, v, batch_first=False)
def test_multihead_attention(self):
    x = torch.randn(16, 24, 32)
    layer = DPMultiheadAttention(32, 1)
    self._check_one_layer(layer, x, x, x, batch_first=False)
    layer = DPMultiheadAttention(32, 1, bias=True, add_bias_kv=True, dropout=0.05)
    self._check_one_layer(layer, x, x, x, batch_first=False)
    layer = DPMultiheadAttention(32, 1, bias=True, add_bias_kv=True)
    self._check_one_layer(layer, x, x, x, batch_first=False)
    layer = DPMultiheadAttention(32, 1, bias=True, add_bias_kv=True, add_zero_attn=True)
    self._check_one_layer(layer, x, x, x, batch_first=False)
    q = torch.randn(16, 24, 32)
    k = torch.randn(20, 24, 28)
    v = torch.randn(20, 24, 28)
    layer = DPMultiheadAttention(32, 1, bias=True, add_bias_kv=True, add_zero_attn=True, kdim=28, vdim=28)
    self._check_one_layer(layer, q, k, v, batch_first=False)