<jupyter_start><jupyter_text>LSTM - text tagger _ todo<jupyter_code># Author: <NAME>

import numpy as np
import torch
import torch.autograd as autograd
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

torch.manual_seed(1)
# define model
lstm = nn.LSTM(3, 3)  # Input dim is 3, output dim is 3
# initialize the hidden state.
hidden = (Variable(torch.randn(1, 1, 3)),
          Variable(torch.randn((1, 1, 3))))

# make a sequence of length 5
inputs = [Variable(torch.randn((1, 3))) for _ in range(5)]  
print("inputs:\n", inputs)

# step by step thru input
for i in inputs:
    # Step through the sequence one element at a time.
    # after each step, hidden contains the hidden state.
    out, hidden = lstm(i.view(1, 1, -1), hidden)

print("out:\n", out)        # all of the hidden states throughout the sequence
print("hidden:\n", hidden)  # the last hidden state, to continue the sequence and backpropagate
# alternatively

# initialize the hidden state - as above
hidden = (Variable(torch.randn(1, 1, 3)),
          Variable(torch.randn((1, 1, 3))))

# make a sequence of length 5
inputs = [Variable(torch.randn((1, 3))) for _ in range(5)]  
print("inputs:\n", inputs)

# concatenate the entire sequence all at once
inputs = torch.cat(inputs).view(len(inputs), 1, -1)
out, hidden = lstm(inputs, hidden)

print("out:\n", out)        # all of the hidden states throughout the sequence
print("hidden:\n", hidden)  # the last hidden state, to continue the sequence and backpropagate
# prepare data
def prepare_sequence(seq, to_ix):
    idxs = [to_ix[w] for w in seq]
    tensor = torch.LongTensor(idxs)
    return autograd.Variable(tensor)


training_data = [
    ("The dog ate the apple".split(), ["DET", "NN", "V", "DET", "NN"]),
    ("Everybody read that book".split(), ["NN", "V", "DET", "NN"])
]
word_to_ix = {}
for sent, tags in training_data:
    for word in sent:
        if word not in word_to_ix:
            word_to_ix[word] = len(word_to_ix)
print(word_to_ix)
tag_to_ix = {"DET": 0, "NN": 1, "V": 2}

# These will usually be more like 32 or 64 dimensional.
# We will keep them small, so we can see how the weights change as we train.
EMBEDDING_DIM = 6
HIDDEN_DIM = 6
#
class LSTMTagger(nn.Module):

    def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size):
        super(LSTMTagger, self).__init__()
        self.hidden_dim = hidden_dim

        self.word_embeddings = nn.Embedding(vocab_size, embedding_dim)

        # The LSTM takes word embeddings as inputs, and outputs hidden states
        # with dimensionality hidden_dim.
        self.lstm = nn.LSTM(embedding_dim, hidden_dim)

        # The linear layer that maps from hidden state space to tag space
        self.hidden2tag = nn.Linear(hidden_dim, tagset_size)
        self.hidden = self.init_hidden()

    def init_hidden(self):
        # Before we've done anything, we dont have any hidden state.
        # Refer to the Pytorch documentation to see exactly
        # why they have this dimensionality.
        # The axes semantics are (num_layers, minibatch_size, hidden_dim)
        return (Variable(torch.zeros(1, 1, self.hidden_dim)),
                Variable(torch.zeros(1, 1, self.hidden_dim)))

    def forward(self, sentence):
        embeds = self.word_embeddings(sentence)
        embeds = embeds.view(len(sentence), 1, -1)   # [sequence, batch, hidden]
        lstm_out, self.hidden = self.lstm(embeds, self.hidden)
        lstm_out = lstm_out.view(len(sentence), -1)  # [sequence, inferred dim]
        tag_space = self.hidden2tag(lstm_out)
        tag_scores = F.log_softmax(tag_space, dim=0)
        return tag_scores
    
#
model = LSTMTagger(EMBEDDING_DIM, HIDDEN_DIM, len(word_to_ix), len(tag_to_ix))
loss_function = nn.NLLLoss()
optimizer = optim.SGD(model.parameters(), lr=0.1)

# See what the scores are before training
# Note that element i,j of the output is the score for tag j for word i.
inputs = prepare_sequence(training_data[0][0], word_to_ix)
tag_scores = model(inputs)
print(tag_scores)

def dict_keys(log_scores, dict, crit = "max"):
    """find index of min/max in array and get corresponding dictionary key"""
    log_scores = np.array(log_scores.data)
    if crit == "min":
        amix = [np.argmin(log_scores[i]) for i in range(len(log_scores))]
    else:
        amix = [np.argmax(log_scores[i]) for i in range(len(log_scores))]
    # find tag keys using argmax index and tag values
    keys = [list(dict.keys())[list(dict.values()).index(amix[i])] 
     for i in range(len(tag_scores))]
    return keys

dict_keys(tag_scores, tag_to_ix, "max")
for epoch in range(300):  # again, normally you would NOT do 300 epochs, it is toy data
    for sentence, tags in training_data:
        # Step 1. Remember that Pytorch accumulates gradients.
        # We need to clear them out before each instance
        model.zero_grad()

        # Also, we need to clear out the hidden state of the LSTM,
        # detaching it from its history on the last instance.
        model.hidden = model.init_hidden()

        # Step 2. Get our inputs ready for the network, that is, turn them into
        # Variables of word indices.
        sentence_in = prepare_sequence(sentence, word_to_ix)
        targets = prepare_sequence(tags, tag_to_ix)

        # Step 3. Run our forward pass.
        tag_scores = model(sentence_in)

        # Step 4. Compute the loss, gradients, and update the parameters by
        #  calling optimizer.step()
        loss = loss_function(tag_scores, targets)
        loss.backward()
        optimizer.step()
# See what the scores are after training
inputs = prepare_sequence(training_data[0][0], word_to_ix)
tag_scores = model(inputs)
print(tag_scores)
# The sentence is "the dog ate the apple". 
# out: [i, j] word i, tag j => max_j for i in sentence is; [0 1 2 0 1]
# Which is DET NOUN VERB DET NOUN, the correct sequence!


def dict_keys(log_scores, dict, crit = "max"):
    """find index of min/max in array and get corresponding dictionary key"""
    log_scores = np.array(log_scores.data)
    if crit == "min":
        amix = [np.argmin(log_scores[i]) for i in range(len(log_scores))]
    else:
        amix = [np.argmax(log_scores[i]) for i in range(len(log_scores))]
    # find tag keys using argmax index and tag values
    keys = [list(dict.keys())[list(dict.values()).index(amix[i])] 
     for i in range(len(tag_scores))]
    return keys

dict_keys(tag_scores, tag_to_ix, "max")


# TODO implement char level LSTM
class LSTMTagger2(nn.Module):

    def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size):
        super(LSTMTagger, self).__init__()
        self.hidden_dim = hidden_dim

        self.word_embeddings = nn.Embedding(vocab_size, embedding_dim)
        self.char_embeddings = nn.Embedding(128, vocab_size, embedding_dim)
        
        # The LSTM takes word embeddings as inputs, and outputs hidden states
        # with dimensionality hidden_dim.
        self.lstm_w = nn.LSTM(embedding_dim, hidden_dim)
        self.lstm_c = nn.LSTM(embedding_dim, embedding_dim, hidden_dim)
        
        # The linear layer that maps from hidden state space to tag space
        self.hidden2tag = nn.Linear(hidden_dim, tagset_size)
        self.hidden = self.init_hidden()

    def init_hidden(self):
        # Before we've done anything, we dont have any hidden state.
        # Refer to the Pytorch documentation to see exactly
        # why they have this dimensionality.
        # The axes semantics are (num_layers, minibatch_size, hidden_dim)
        return (Variable(torch.zeros(1, 1, self.hidden_dim)),
                Variable(torch.zeros(1, 1, self.hidden_dim)))

    def forward(self, sentence):
        char = sentence.view(len(), len(sentence), 1, -1)
        embeds = self.word_embeddings(sentence)
        embed_c = self.char_embeddings([char for char in sentence])
        
        embeds = embeds.view(len(sentence), 1, -1)   # [sequence, batch, hidden]
        embeds = embeds.view(len(sentence), 1, -1)   # [sequence, batch, hidden]
        
        w_out, self.hidden = self.lstm_w(embeds, self.hidden)
        c_out, self.hidden = self.lstm_c(w_out, self.hidden)
        
        # lstm_out, self.hidden = self.lstm(embeds, self.hidden)
        
        lstm_out = c_out.view(len(sentence), -1)  # [sequence, inferred dim]
        tag_space = self.hidden2tag(lstm_out)
        tag_scores = F.log_softmax(tag_space, dim=0)
        return tag_scores<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1077788191190253, 'fraction_numerical': 0.010309278350515464, 'mean_word_length': 3.2095660749506902, 'pattern_counts': {'":': 5, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '432890', 'license': 'MIT', 'n_tokens_mistral': 2883, 'n_tokens_neox': 2745, 'n_words': 859, 'path': '_RNN/pytorch_lstm_text_tagger_todo.ipynb', 'repo_name': 'aixpact/deep-learning', 'size': '8544'}