def __init__(self, args, dictionary, embed_tokens, no_encoder_attn=False):
    super().__init__(dictionary)
    self.register_buffer('version', torch.Tensor([3]))
    self.dropout = args.dropout
    self.share_input_output_embed = args.share_decoder_input_output_embed
    input_embed_dim = embed_tokens.embedding_dim
    embed_dim = args.decoder_embed_dim
    self.output_embed_dim = args.decoder_output_dim
    padding_idx = embed_tokens.padding_idx
    self.max_target_positions = args.max_target_positions
    self.embed_tokens = embed_tokens
    self.embed_scale = math.sqrt(embed_dim)
    self.project_in_dim = (Linear(input_embed_dim, embed_dim, bias=False) if (embed_dim != input_embed_dim) else None)
    self.embed_positions = (PositionalEmbedding(args.max_target_positions, embed_dim, padding_idx, learned=args.decoder_learned_pos) if (not args.no_token_positional_embeddings) else None)
    bert_gates = getattr(args, 'bert_gates', [1, 1, 1, 1, 1, 1])
    bert_gates = [(x == 1) for x in bert_gates]
    assert (len(bert_gates) == args.decoder_layers)
    print('bert_gates', bert_gates)
    self.layers = nn.ModuleList([])
    decoder_no_bert = getattr(args, 'decoder_no_bert', False)
    if decoder_no_bert:
        self.layers.extend([TransformerStandardDecoderLayer(args, no_encoder_attn, bert_gate=bert_gates[i]) for i in range(args.decoder_layers)])
    else:
        self.layers.extend([TransformerDecoderLayer(args, no_encoder_attn, bert_gate=bert_gates[i]) for i in range(args.decoder_layers)])
    self.adaptive_softmax = None
    self.project_out_dim = (Linear(embed_dim, self.output_embed_dim, bias=False) if ((embed_dim != self.output_embed_dim) and (not args.tie_adaptive_weights)) else None)
    if (args.adaptive_softmax_cutoff is not None):
        self.adaptive_softmax = AdaptiveSoftmax(len(dictionary), self.output_embed_dim, options.eval_str_list(args.adaptive_softmax_cutoff, type=int), dropout=args.adaptive_softmax_dropout, adaptive_inputs=(embed_tokens if args.tie_adaptive_weights else None), factor=args.adaptive_softmax_factor, tie_proj=args.tie_adaptive_proj)
    elif (not self.share_input_output_embed):
        self.embed_out = nn.Parameter(torch.Tensor(len(dictionary), self.output_embed_dim))
        nn.init.normal_(self.embed_out, mean=0, std=(self.output_embed_dim ** (- 0.5)))
    if (args.decoder_normalize_before and (not getattr(args, 'no_decoder_final_norm', False))):
        self.layer_norm = LayerNorm(embed_dim)
    else:
        self.layer_norm = None
def __init__(self, args, dictionary, embed_tokens, no_encoder_attn=False):
    super().__init__(dictionary)
    self.register_buffer('version', torch.Tensor([3]))
    self.dropout = args.dropout
    self.share_input_output_embed = args.share_decoder_input_output_embed
    input_embed_dim = embed_tokens.embedding_dim
    embed_dim = args.decoder_embed_dim
    self.output_embed_dim = args.decoder_output_dim
    padding_idx = embed_tokens.padding_idx
    self.max_target_positions = args.max_target_positions
    self.embed_tokens = embed_tokens
    self.embed_scale = math.sqrt(embed_dim)
    self.project_in_dim = (Linear(input_embed_dim, embed_dim, bias=False) if (embed_dim != input_embed_dim) else None)
    self.embed_positions = (PositionalEmbedding(args.max_target_positions, embed_dim, padding_idx, learned=args.decoder_learned_pos) if (not args.no_token_positional_embeddings) else None)
    bert_gates = getattr(args, 'bert_gates', [1, 1, 1, 1, 1, 1])
    bert_gates = [(x == 1) for x in bert_gates]
    assert (len(bert_gates) == args.decoder_layers)
    print('bert_gates', bert_gates)
    self.layers = nn.ModuleList([])
    decoder_no_bert = getattr(args, 'decoder_no_bert', False)
    if decoder_no_bert:
        self.layers.extend([TransformerStandardDecoderLayer(args, no_encoder_attn, bert_gate=bert_gates[i]) for i in range(args.decoder_layers)])
    else:
        self.layers.extend([TransformerDecoderLayer(args, no_encoder_attn, bert_gate=bert_gates[i]) for i in range(args.decoder_layers)])
    self.adaptive_softmax = None
    self.project_out_dim = (Linear(embed_dim, self.output_embed_dim, bias=False) if ((embed_dim != self.output_embed_dim) and (not args.tie_adaptive_weights)) else None)
    if (args.adaptive_softmax_cutoff is not None):
        self.adaptive_softmax = AdaptiveSoftmax(len(dictionary), self.output_embed_dim, options.eval_str_list(args.adaptive_softmax_cutoff, type=int), dropout=args.adaptive_softmax_dropout, adaptive_inputs=(embed_tokens if args.tie_adaptive_weights else None), factor=args.adaptive_softmax_factor, tie_proj=args.tie_adaptive_proj)
    elif (not self.share_input_output_embed):
        self.embed_out = nn.Parameter(torch.Tensor(len(dictionary), self.output_embed_dim))
        nn.init.normal_(self.embed_out, mean=0, std=(self.output_embed_dim ** (- 0.5)))
    if (args.decoder_normalize_before and (not getattr(args, 'no_decoder_final_norm', False))):
        self.layer_norm = LayerNorm(embed_dim)
    else:
        self.layer_norm = None