def forward(self, src_tokens, src_lengths, bert_encoder_out):
    "\n        Args:\n            src_tokens (LongTensor): tokens in the source language of shape\n                `(batch, src_len)`\n            src_lengths (torch.LongTensor): lengths of each source sentence of\n                shape `(batch)`\n\n        Returns:\n            dict:\n                - **encoder_out** (Tensor): the last encoder layer's output of\n                  shape `(src_len, batch, embed_dim)`\n                - **encoder_padding_mask** (ByteTensor): the positions of\n                  padding elements of shape `(batch, src_len)`\n        "
    x = (self.embed_scale * self.embed_tokens(src_tokens))
    if (self.embed_positions is not None):
        x += self.embed_positions(src_tokens)
    x = F.dropout(x, p=self.dropout, training=self.training)
    '\n        mask_output = self.mask(src_tokens , x)\n        p = mask_output\n        p = p.transpose(0, 1)\n        t_p = torch.argsort(p,dim=1)\n        ratio = 0.2\n        self.ratio = ratio\n        p_mask = torch.where(t_p<t_p.size(1)*ratio,torch.zeros_like(p),torch.ones_like(p))\n        self.p_mask = p_mask\n        p_mask = p_mask.unsqueeze(-1).transpose(0,1)\n\n        self.mask_output = p\n    \n        \n        if self.training:\n            x = x * p_mask.detach()\n        else:\n            x = x\n        ###########\n        ###########\n        ###########\n        # t_p[t_p>t_p.size*ratio] = 1\n        # t_p[t_p<=t_p.size*ratio] = 0\n        # t_p.permute(1,0)\n        \n        \n        # model.encoder.mask_output \n        '
    x = x.transpose(0, 1)
    encoder_padding_mask = src_tokens.eq(self.padding_idx)
    if (not encoder_padding_mask.any()):
        encoder_padding_mask = None
    for layer in self.layers:
        x = layer(x, encoder_padding_mask, bert_encoder_out['bert_encoder_out'], bert_encoder_out['bert_encoder_padding_mask'])
    if self.layer_norm:
        x = self.layer_norm(x)
    '\n        self.encoder_vocab_output = self.encodeMLM(src_tokens, src_lengths, bert_encoder_out)\n        '
    '\n        ##########################\n        if self.i%1==0:\n            import scipy.io as scio\n            self.encoder_vocab_output = self.encodeMLM(src_tokens, src_lengths, bert_encoder_out)\n            scio.savemat("/home/iojhui/bert-nmt/data"+str(self.i)+".mat", {\'mask_output\':self.mask_output.detach().cpu().numpy(),"src_tokens":src_tokens.cpu().numpy()})\n            \n\n\n        self.i+=1\n        ########################\n        '
    return {'encoder_out': x, 'encoder_padding_mask': encoder_padding_mask}
Args:
    src_tokens (LongTensor): tokens in the source language of shape
        `(batch, src_len)`
    src_lengths (torch.LongTensor): lengths of each source sentence of
        shape `(batch)`

Returns:
    dict:
        - **encoder_out** (Tensor): the last encoder layer's output of
          shape `(src_len, batch, embed_dim)`
        - **encoder_padding_mask** (ByteTensor): the positions of
          padding elements of shape `(batch, src_len)`
def forward(self, src_tokens, src_lengths, bert_encoder_out):
    "\n        Args:\n            src_tokens (LongTensor): tokens in the source language of shape\n                `(batch, src_len)`\n            src_lengths (torch.LongTensor): lengths of each source sentence of\n                shape `(batch)`\n\n        Returns:\n            dict:\n                - **encoder_out** (Tensor): the last encoder layer's output of\n                  shape `(src_len, batch, embed_dim)`\n                - **encoder_padding_mask** (ByteTensor): the positions of\n                  padding elements of shape `(batch, src_len)`\n        "
    x = (self.embed_scale * self.embed_tokens(src_tokens))
    if (self.embed_positions is not None):
        x += self.embed_positions(src_tokens)
    x = F.dropout(x, p=self.dropout, training=self.training)
    '\n        mask_output = self.mask(src_tokens , x)\n        p = mask_output\n        p = p.transpose(0, 1)\n        t_p = torch.argsort(p,dim=1)\n        ratio = 0.2\n        self.ratio = ratio\n        p_mask = torch.where(t_p<t_p.size(1)*ratio,torch.zeros_like(p),torch.ones_like(p))\n        self.p_mask = p_mask\n        p_mask = p_mask.unsqueeze(-1).transpose(0,1)\n\n        self.mask_output = p\n    \n        \n        if self.training:\n            x = x * p_mask.detach()\n        else:\n            x = x\n        ###########\n        ###########\n        ###########\n        # t_p[t_p>t_p.size*ratio] = 1\n        # t_p[t_p<=t_p.size*ratio] = 0\n        # t_p.permute(1,0)\n        \n        \n        # model.encoder.mask_output \n        '
    x = x.transpose(0, 1)
    encoder_padding_mask = src_tokens.eq(self.padding_idx)
    if (not encoder_padding_mask.any()):
        encoder_padding_mask = None
    for layer in self.layers:
        x = layer(x, encoder_padding_mask, bert_encoder_out['bert_encoder_out'], bert_encoder_out['bert_encoder_padding_mask'])
    if self.layer_norm:
        x = self.layer_norm(x)
    '\n        self.encoder_vocab_output = self.encodeMLM(src_tokens, src_lengths, bert_encoder_out)\n        '
    '\n        ##########################\n        if self.i%1==0:\n            import scipy.io as scio\n            self.encoder_vocab_output = self.encodeMLM(src_tokens, src_lengths, bert_encoder_out)\n            scio.savemat("/home/iojhui/bert-nmt/data"+str(self.i)+".mat", {\'mask_output\':self.mask_output.detach().cpu().numpy(),"src_tokens":src_tokens.cpu().numpy()})\n            \n\n\n        self.i+=1\n        ########################\n        '
    return {'encoder_out': x, 'encoder_padding_mask': encoder_padding_mask}