def __init__(self, channels, num_heads=1, num_head_channels=(- 1), use_checkpoint=False, use_new_attention_order=False, cross=False, enc_attn_dim=None, norm_type='group', dropout=0.0):
    super().__init__()
    self.channels = channels
    if (num_head_channels == (- 1)):
        self.num_heads = num_heads
    else:
        assert ((channels % num_head_channels) == 0), f'q,k,v channels {channels} is not divisible by num_head_channels {num_head_channels}'
        self.num_heads = (channels // num_head_channels)
    self.use_checkpoint = use_checkpoint
    self.norm = (normalization(channels) if (norm_type == 'group') else LayerNorm32(channels))
    qkv_dim = ((channels * 3) if (not cross) else channels)
    self.qkv = conv_nd(1, channels, qkv_dim, 1)
    self.attention = QKVAttentionLegacy(self.num_heads, dropout=dropout)
    self.proj_out = conv_nd(1, channels, channels, 1)
    if (norm_type == 'group'):
        self.proj_out = zero_module(self.proj_out)
    self.dropout = nn.Dropout(p=dropout)
    if (cross and (channels != enc_attn_dim)):
        self.adjust_kv_dim = conv_nd(1, enc_attn_dim, channels, 1)
def __init__(self, channels, num_heads=1, num_head_channels=(- 1), use_checkpoint=False, use_new_attention_order=False, cross=False, enc_attn_dim=None, norm_type='group', dropout=0.0):
    super().__init__()
    self.channels = channels
    if (num_head_channels == (- 1)):
        self.num_heads = num_heads
    else:
        assert ((channels % num_head_channels) == 0), f'q,k,v channels {channels} is not divisible by num_head_channels {num_head_channels}'
        self.num_heads = (channels // num_head_channels)
    self.use_checkpoint = use_checkpoint
    self.norm = (normalization(channels) if (norm_type == 'group') else LayerNorm32(channels))
    qkv_dim = ((channels * 3) if (not cross) else channels)
    self.qkv = conv_nd(1, channels, qkv_dim, 1)
    self.attention = QKVAttentionLegacy(self.num_heads, dropout=dropout)
    self.proj_out = conv_nd(1, channels, channels, 1)
    if (norm_type == 'group'):
        self.proj_out = zero_module(self.proj_out)
    self.dropout = nn.Dropout(p=dropout)
    if (cross and (channels != enc_attn_dim)):
        self.adjust_kv_dim = conv_nd(1, enc_attn_dim, channels, 1)