<jupyter_start><jupyter_text>Loss function<jupyter_code># target_padded
vocab.tgt['<pad>']
target_masks=(target_padded != vocab.tgt['<pad>']).float()<jupyter_output><empty_output><jupyter_text>Question 1e<jupyter_code>enc_hiddens.shape
hidden_size = HIDDEN_SIZE
att_projection = nn.Linear(hidden_size*2, hidden_size, bias=False)
att_projection(enc_hiddens).shape
model_embeddings.target(target_padded)<jupyter_output><empty_output><jupyter_text>**target explanation**<jupyter_code>target_padded.shape
target_padded = target_padded[:-1]
target_padded.shape
[len(t) for t in tgt_sents]
mytgt = vocab.tgt.to_input_tensor(tgt_sents, device='cpu')
# tgt_sents #english while src_sents is spanish
# mytgt = target_padded
# target_padded
#tgt_sents[0] ['<s>', 'Let', "'s'", 'start', ...]

#src_sents[0] ['Comencemos', 'por', 'pensar', ...]<jupyter_output><empty_output><jupyter_text>look back to target paddwed<jupyter_code>target_padded.shape
Y = model_embeddings.target(target_padded)
Y.shape
x = torch.split(Y, 1, dim=0)
x[0].size()
x[0] # this is embedding for <s> and you need learn embedding
torch.squeeze(x[0]).shape
Yt.shape
dec_state #(two tensor = dec_init_state)
enc_hiddens.shape #torch.Size([5, 22, 6])
enc_hiddens_proj.shape #torch.Size([5, 22, 3])
enc_hiddens_proj = att_projection(enc_hiddens)
enc_masks.shape #torch.Size([5, 22])
Ybar_t.shape  #torch.Size([5, 6])
o_prev.shape # torch.Size([5, 3])
len(Y_ts)
COUNTER = [0]
def stepFunction(Ybar_t, dec_state, enc_hiddens, enc_hiddens_proj, enc_masks):
   dec_state = torch.load('./sanity_check_en_es_data/step_dec_state_{}.pkl'.format(COUNTER[0]))
   o_t = torch.load('./sanity_check_en_es_data/step_o_t_{}.pkl'.format(COUNTER[0]))
   COUNTER[0]+=1
   return dec_state, o_t, None
model.step = stepFunction
o_prev = torch.zeros(5, 3, device='cpu')
combined_outputs = []
Y = model_embeddings.target(target_padded) # 23, 已经去掉最后一个</s>
Y_ts = torch.split(Y, 1)
for i in range(len(Y_ts)):
#     print (i)
    Y_t = Y_ts[i]
    Y_t = torch.squeeze(Y_t)
    Ybar_t = torch.cat((Y_t, o_prev), 1)
    dec_state, o_t, _ = model.step(Ybar_t, dec_state, enc_hiddens, enc_hiddens_proj,
                                                        enc_masks)
#     o_prev = o_t
    combined_outputs.append(o_t)
    o_prev = o_t
#     print (o_t.shape)
combined_outputs = torch.stack(combined_outputs, dim=0)
Y.shape
len(Y_ts)
len(combined_outputs)
# combined_outputs consistent with combined_outputs_target from loading output<jupyter_output><empty_output><jupyter_text>**loading output**<jupyter_code>dec_init_state = torch.load('./sanity_check_en_es_data/dec_init_state.pkl')
enc_hiddens = torch.load('./sanity_check_en_es_data/enc_hiddens.pkl')
enc_masks = torch.load('./sanity_check_en_es_data/enc_masks.pkl')
target_padded = torch.load('./sanity_check_en_es_data/target_padded.pkl')

enc_hiddens.shape
target_padded.shape #correct with tgt_sents
combined_outputs_target = torch.load('./sanity_check_en_es_data/combined_outputs.pkl')
print(combined_outputs_target.shape)
enc_hiddens.shape
xx = combined_outputs_target.numpy()
len(xx[0]) # 22*5*3
# combined_outputs_pred = model.decode(enc_hiddens, enc_masks, dec_init_state, target_padded)<jupyter_output><empty_output><jupyter_text>Question 1f<jupyter_code>def reinitialize_layers(model):
    """ Reinitialize the Layer Weights for Sanity Checks.
    """
    def init_weights(m):
        if type(m) == nn.Linear:
            m.weight.data.fill_(0.3)
            if m.bias is not None:
                m.bias.data.fill_(0.1)
        elif type(m) == nn.Embedding:
            m.weight.data.fill_(0.15)
        elif type(m) == nn.Dropout:
            nn.Dropout(DROPOUT_RATE)
    with torch.no_grad():
        model.apply(init_weights)
from typing import List, Tuple, Dict, Set, Union
class ModelEmbeddings(nn.Module): 
    """
    Class that converts input words to their embeddings.
    """
    def __init__(self, embed_size, vocab):
        super(ModelEmbeddings, self).__init__()
        self.embed_size = embed_size

        # default values
        self.source = None
        self.target = None

        src_pad_token_idx = vocab.src['<pad>']
        tgt_pad_token_idx = vocab.tgt['<pad>']

        self.source = nn.Embedding(len(vocab.src), embed_size, src_pad_token_idx)
        self.target = nn.Embedding(len(vocab.tgt), embed_size, tgt_pad_token_idx)

class NMT(nn.Module):
    """ Simple Neural Machine Translation Model:
        - Bidrectional LSTM Encoder
        - Unidirection LSTM Decoder
        - Global Attention Model (Luong, et al. 2015)
    """
    def __init__(self, embed_size, hidden_size, vocab, dropout_rate=0.2):
        super(NMT, self).__init__()
        self.model_embeddings = ModelEmbeddings(embed_size, vocab)
        self.hidden_size = hidden_size
        self.dropout_rate = dropout_rate
        self.vocab = vocab

        # default values
        self.encoder = None 
        self.decoder = None
        self.h_projection = None
        self.c_projection = None
        self.att_projection = None
        self.combined_output_projection = None
        self.target_vocab_projection = None
        self.dropout = None
        # For sanity check only, not relevant to implementation
        self.gen_sanity_check = False
        self.counter = 0

    def decode(self, enc_hiddens: torch.Tensor, enc_masks: torch.Tensor,
                dec_init_state: Tuple[torch.Tensor, torch.Tensor], target_padded: torch.Tensor) -> torch.Tensor:
        """Compute combined output vectors for a batch.

        @param enc_hiddens (Tensor): Hidden states (b, src_len, h*2), where
                                     b = batch size, src_len = maximum source sentence length, h = hidden size.
        @param enc_masks (Tensor): Tensor of sentence masks (b, src_len), where
                                     b = batch size, src_len = maximum source sentence length.
        @param dec_init_state (tuple(Tensor, Tensor)): Initial state and cell for decoder
        @param target_padded (Tensor): Gold-standard padded target sentences (tgt_len, b), where
                                       tgt_len = maximum target sentence length, b = batch size. 

        @returns combined_outputs (Tensor): combined output tensor  (tgt_len, b,  h), where
                                        tgt_len = maximum target sentence length, b = batch_size,  h = hidden size
        """
        # Chop of the <END> token for max length sentences.
        target_padded = target_padded[:-1]

        # Initialize the decoder state (hidden and cell)
        dec_state = dec_init_state

        # Initialize previous combined output vector o_{t-1} as zero
        batch_size = enc_hiddens.size(0)
        o_prev = torch.zeros(batch_size, self.hidden_size, device='cpu')

        # Initialize a list we will use to collect the combined output o_t on each step
        combined_outputs = []

        enc_hiddens_proj = self.att_projection(enc_hiddens)
        Y = self.model_embeddings.target(target_padded)
        Y_ts = torch.split(Y, 1)
        for Y_t in Y_ts:
            Y_t = torch.squeeze(Y_t)
            Ybar_t = torch.cat((Y_t, o_prev), 1)
            dec_state, o_t, _ = self.step(Ybar_t, dec_state, enc_hiddens, enc_hiddens_proj,
                                                        enc_masks)
            combined_outputs.append(o_t.tolist())
            o_prev = o_t
        combined_outputs = torch.stack(combined_outputs, dim=0)


def words2indices(sents):
    """ Convert list of words or list of sentences of words
    into list or list of list of indices.
    @param sents (list[str] or list[list[str]]): sentence(s) in words
    @return word_ids (list[int] or list[list[int]]): sentence(s) in indices
    """
    if type(sents[0]) == list:
        return [[[w] for w in s] for s in sents]
    else:
        return [[w] for w in sents]
t = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
t.gather(0, torch.tensor([[2,1, 0]]))
model
model.train()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11437867601051183, 'fraction_numerical': 0.009886121887123013, 'mean_word_length': 3.230809952355744, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '435941', 'license': 'Apache-2.0', 'n_tokens_mistral': 2845, 'n_tokens_neox': 2778, 'n_words': 592, 'path': 'Assignments/assignment4/stxx/.ipynb_checkpoints/play-checkpoint.ipynb', 'repo_name': 'stxxllbu/CS224n-winter-together', 'size': '7991'}