def __init__(self, args, dictionary, embed_tokens):
    super().__init__(dictionary)
    self.register_buffer('version', torch.Tensor([3]))
    self.dropout = args.dropout
    self.output_mask = nn.Softmax(dim=0)
    self.t_layer = nn.Linear(512, 1)
    self.output_vocab_linear = nn.Linear(512, embed_tokens.num_embeddings)
    embed_dim = embed_tokens.embedding_dim
    self.padding_idx = embed_tokens.padding_idx
    self.max_source_positions = args.max_source_positions
    self.embed_tokens = embed_tokens
    self.embed_scale = math.sqrt(embed_dim)
    self.embed_positions = (PositionalEmbedding(args.max_source_positions, embed_dim, self.padding_idx, learned=args.encoder_learned_pos) if (not args.no_token_positional_embeddings) else None)
    bert_gates = getattr(args, 'bert_gates', [1, 1, 1, 1, 1, 1])
    bert_gates = [(x == 1) for x in bert_gates]
    assert (len(bert_gates) == args.encoder_layers)
    self.layers = nn.ModuleList([])
    self.layers.extend([TransformerS2EncoderLayer(args, bert_gate=bert_gates[i]) for i in range(args.encoder_layers)])
    if args.encoder_normalize_before:
        self.layer_norm = LayerNorm(embed_dim)
    else:
        self.layer_norm = None
    self.mask_embedding = nn.init.normal_(nn.Parameter(torch.zeros((1, embed_dim))))
    self.mask_layers = nn.ModuleList([])
    self.mask_layers.extend([TransformerEncoderLayer(args) for i in range(2)])
    if args.encoder_normalize_before:
        self.mask_layer_norm = LayerNorm(embed_dim)
    else:
        self.layer_norm = None
    '    \n        self.x = None\n        self.unmask_output =  None\n        self.mask_output =  None\n        self.encoder_vocab_output =  None\n\n        self.backwards = 0\n        '
    self.i = 0
def __init__(self, args, dictionary, embed_tokens):
    super().__init__(dictionary)
    self.register_buffer('version', torch.Tensor([3]))
    self.dropout = args.dropout
    self.output_mask = nn.Softmax(dim=0)
    self.t_layer = nn.Linear(512, 1)
    self.output_vocab_linear = nn.Linear(512, embed_tokens.num_embeddings)
    embed_dim = embed_tokens.embedding_dim
    self.padding_idx = embed_tokens.padding_idx
    self.max_source_positions = args.max_source_positions
    self.embed_tokens = embed_tokens
    self.embed_scale = math.sqrt(embed_dim)
    self.embed_positions = (PositionalEmbedding(args.max_source_positions, embed_dim, self.padding_idx, learned=args.encoder_learned_pos) if (not args.no_token_positional_embeddings) else None)
    bert_gates = getattr(args, 'bert_gates', [1, 1, 1, 1, 1, 1])
    bert_gates = [(x == 1) for x in bert_gates]
    assert (len(bert_gates) == args.encoder_layers)
    self.layers = nn.ModuleList([])
    self.layers.extend([TransformerS2EncoderLayer(args, bert_gate=bert_gates[i]) for i in range(args.encoder_layers)])
    if args.encoder_normalize_before:
        self.layer_norm = LayerNorm(embed_dim)
    else:
        self.layer_norm = None
    self.mask_embedding = nn.init.normal_(nn.Parameter(torch.zeros((1, embed_dim))))
    self.mask_layers = nn.ModuleList([])
    self.mask_layers.extend([TransformerEncoderLayer(args) for i in range(2)])
    if args.encoder_normalize_before:
        self.mask_layer_norm = LayerNorm(embed_dim)
    else:
        self.layer_norm = None
    '    \n        self.x = None\n        self.unmask_output =  None\n        self.mask_output =  None\n        self.encoder_vocab_output =  None\n\n        self.backwards = 0\n        '
    self.i = 0