def __init__(self, args, no_encoder_attn=False, add_bias_kv=False, add_zero_attn=False, bert_gate=True):
    super().__init__()
    self.embed_dim = args.decoder_embed_dim
    self.self_attn = MultiheadAttention(embed_dim=self.embed_dim, num_heads=args.decoder_attention_heads, dropout=args.attention_dropout, add_bias_kv=add_bias_kv, add_zero_attn=add_zero_attn, self_attention=True)
    self.dropout = args.dropout
    self.activation_fn = utils.get_activation_fn(activation=getattr(args, 'activation_fn', 'relu'))
    self.activation_dropout = getattr(args, 'activation_dropout', 0)
    if (self.activation_dropout == 0):
        self.activation_dropout = getattr(args, 'relu_dropout', 0)
    self.normalize_before = args.decoder_normalize_before
    export = getattr(args, 'char_inputs', False)
    self.self_attn_layer_norm = LayerNorm(self.embed_dim, export=export)
    if no_encoder_attn:
        self.encoder_attn = None
        self.encoder_attn_layer_norm = None
    else:
        self.encoder_attn = MultiheadAttention(self.embed_dim, args.decoder_attention_heads, dropout=args.attention_dropout, encoder_decoder_attention=True)
        self.encoder_attn_layer_norm = LayerNorm(self.embed_dim, export=export)
    self.fc1 = Linear(self.embed_dim, args.decoder_ffn_embed_dim)
    self.fc2 = Linear(args.decoder_ffn_embed_dim, self.embed_dim)
    self.final_layer_norm = LayerNorm(self.embed_dim, export=export)
    self.need_attn = True
    self.onnx_trace = False
    self.encoder_ratio = args.encoder_ratio
    self.bert_ratio = args.bert_ratio
    if (not bert_gate):
        self.bert_ratio = 0.0
    self.encoder_bert_dropout = getattr(args, 'encoder_bert_dropout', False)
    self.encoder_bert_dropout_ratio = getattr(args, 'encoder_bert_dropout_ratio', 0.25)
    assert ((self.encoder_bert_dropout_ratio >= 0.0) and (self.encoder_bert_dropout_ratio <= 0.5))
    self.encoder_bert_mixup = getattr(args, 'encoder_bert_mixup', False)
def __init__(self, args, no_encoder_attn=False, add_bias_kv=False, add_zero_attn=False, bert_gate=True):
    super().__init__()
    self.embed_dim = args.decoder_embed_dim
    self.self_attn = MultiheadAttention(embed_dim=self.embed_dim, num_heads=args.decoder_attention_heads, dropout=args.attention_dropout, add_bias_kv=add_bias_kv, add_zero_attn=add_zero_attn, self_attention=True)
    self.dropout = args.dropout
    self.activation_fn = utils.get_activation_fn(activation=getattr(args, 'activation_fn', 'relu'))
    self.activation_dropout = getattr(args, 'activation_dropout', 0)
    if (self.activation_dropout == 0):
        self.activation_dropout = getattr(args, 'relu_dropout', 0)
    self.normalize_before = args.decoder_normalize_before
    export = getattr(args, 'char_inputs', False)
    self.self_attn_layer_norm = LayerNorm(self.embed_dim, export=export)
    if no_encoder_attn:
        self.encoder_attn = None
        self.encoder_attn_layer_norm = None
    else:
        self.encoder_attn = MultiheadAttention(self.embed_dim, args.decoder_attention_heads, dropout=args.attention_dropout, encoder_decoder_attention=True)
        self.encoder_attn_layer_norm = LayerNorm(self.embed_dim, export=export)
    self.fc1 = Linear(self.embed_dim, args.decoder_ffn_embed_dim)
    self.fc2 = Linear(args.decoder_ffn_embed_dim, self.embed_dim)
    self.final_layer_norm = LayerNorm(self.embed_dim, export=export)
    self.need_attn = True
    self.onnx_trace = False
    self.encoder_ratio = args.encoder_ratio
    self.bert_ratio = args.bert_ratio
    if (not bert_gate):
        self.bert_ratio = 0.0
    self.encoder_bert_dropout = getattr(args, 'encoder_bert_dropout', False)
    self.encoder_bert_dropout_ratio = getattr(args, 'encoder_bert_dropout_ratio', 0.25)
    assert ((self.encoder_bert_dropout_ratio >= 0.0) and (self.encoder_bert_dropout_ratio <= 0.5))
    self.encoder_bert_mixup = getattr(args, 'encoder_bert_mixup', False)