def __init__(self, args):
    super().__init__()
    self.embed_dim = args.encoder_embed_dim
    self.self_attn = MultiheadAttention(self.embed_dim, args.encoder_attention_heads, dropout=args.attention_dropout, self_attention=True)
    self.self_attn_layer_norm = LayerNorm(self.embed_dim)
    self.dropout = args.dropout
    self.activation_fn = utils.get_activation_fn(activation=getattr(args, 'activation_fn', 'relu'))
    self.activation_dropout = getattr(args, 'activation_dropout', 0)
    if (self.activation_dropout == 0):
        self.activation_dropout = getattr(args, 'relu_dropout', 0)
    self.normalize_before = args.encoder_normalize_before
    self.fc1 = Linear(self.embed_dim, args.encoder_ffn_embed_dim)
    self.fc2 = Linear(args.encoder_ffn_embed_dim, self.embed_dim)
    self.final_layer_norm = LayerNorm(self.embed_dim)
def __init__(self, args):
    super().__init__()
    self.embed_dim = args.encoder_embed_dim
    self.self_attn = MultiheadAttention(self.embed_dim, args.encoder_attention_heads, dropout=args.attention_dropout, self_attention=True)
    self.self_attn_layer_norm = LayerNorm(self.embed_dim)
    self.dropout = args.dropout
    self.activation_fn = utils.get_activation_fn(activation=getattr(args, 'activation_fn', 'relu'))
    self.activation_dropout = getattr(args, 'activation_dropout', 0)
    if (self.activation_dropout == 0):
        self.activation_dropout = getattr(args, 'relu_dropout', 0)
    self.normalize_before = args.encoder_normalize_before
    self.fc1 = Linear(self.embed_dim, args.encoder_ffn_embed_dim)
    self.fc2 = Linear(args.encoder_ffn_embed_dim, self.embed_dim)
    self.final_layer_norm = LayerNorm(self.embed_dim)