<jupyter_start><jupyter_text>Preparing Data<jupyter_code>train_data = pd.read_csv('./archive/Train.csv')
test_data = pd.read_csv('./archive/Test.csv')
validation_data = pd.read_csv('./archive/Valid.csv')
train_data.head()<jupyter_output><empty_output><jupyter_text>DownsamplingWe don't need much data to train the model, let's downsample it and train on a small subset.<jupyter_code>train_data = train_data.sample(1000, random_state=1).reset_index(drop=True)
test_data = test_data.sample(200, random_state=1).reset_index(drop=True)
validation_data = validation_data.sample(200, random_state=1).reset_index(drop=True)
train_data.head()
x_tr, y_tr = train_data['text'].values.astype(str), train_data['label'].values
x_val, y_val = validation_data['text'].values.astype(str), validation_data['label'].values
x_test,y_test = test_data['text'].values.astype(str) , test_data['label'].values
print(len(x_tr), len(y_tr))
print(len(x_val), len(y_val))
print(len(x_test), len(y_test))
#Tokenize the sentences
from string import punctuation
#convert all revies to lowercase
x_tr = np.char.lower(x_tr)
x_val = np.char.lower(x_val)
x_test = np.char.lower(x_test)

#remove punctuation
x_tr= [c for c in x_tr if c not in punctuation]
x_val = [c for c in x_val if c not in punctuation]
x_test = [c for c in x_test if c not in punctuation]
def read_glove_vector(glove_vec):
    with open(glove_vec, 'r', encoding='UTF-8') as f:
        words = set()
        word_to_vec_map = {}
        for line in f:
            w_line = line.split()
            curr_word = w_line[0]
            word_to_vec_map[curr_word] = np.array(w_line[1:], dtype=np.float64)



    return word_to_vec_map
glove_embedding =  read_glove_vector('glove.6B.100d.txt')

def create_embedding_matrix(word_index,embedding_dict,dimension):
    embedding_matrix=np.zeros((len(word_index)+1,dimension))
    for word,index in word_index.items():
        if word in embedding_dict:
            embedding_matrix[index]=embedding_dict[word]
    return embedding_matrix
tokenizer = Tokenizer()

#preparing vocabulary
tokenizer.fit_on_texts(list(x_tr))

#converting text into integer sequences
x_tr_seq  = tokenizer.texts_to_sequences(x_tr) 
x_val_seq = tokenizer.texts_to_sequences(x_val)
x_test_seq = tokenizer.texts_to_sequences(x_test)


#padding to prepare sequences of same length
x_tr_pad  = pad_sequences(x_tr_seq, maxlen=1000)
x_val_pad = pad_sequences(x_val_seq, maxlen=1000)
x_test_pad = pad_sequences(x_test_seq,maxlen=1000)

embedding_matrix=create_embedding_matrix(tokenizer.word_index,embedding_dict=glove_embedding,dimension=100)
vocab_size=embedding_matrix.shape[0]
vector_size=embedding_matrix.shape[1]
print(vocab_size)
vector_size<jupyter_output>18828<jupyter_text>We will need the original lengths of each sequence later:<jupyter_code>#length of the reviews
seqlen_train = [len(x) for x in x_tr_seq]
seqlen_val = [len(x) for x in x_val_seq] 
seqlen_test = [len(x) for x in x_test_seq] 
# this might be an issue 
vocab=len(tokenizer.word_index) + 1 #+1 for padding
print(vocab)<jupyter_output>18828<jupyter_text>Creating Tensors  Extract input and output tensors<jupyter_code># x = torch.tensor(train).to(torch.float64)

X_tr = torch.from_numpy(np.array(list(x_tr_pad)))
X_train = torch.tensor(X_tr).to(torch.int64)

y_train = torch.LongTensor(y_tr)
seqlen_train = torch.LongTensor(seqlen_train)
print(f"{X_train.size()},\n{y_train.size()}\n{seqlen_train.size() }\n")
print("tensor dimensions", X_train.dim)

X_v = torch.from_numpy(np.array(list(x_val_pad)))
X_val = torch.tensor(X_v).to(torch.torch.int64)

y_val = torch.LongTensor(y_val)
seqlen_val = torch.LongTensor(seqlen_val)
print(f"{X_val.size() },\n{y_val.size() }\n{seqlen_val.size() }\n")


X_t = torch.from_numpy(np.array(list(x_test_pad)))
X_test = torch.tensor(X_t).to(torch.torch.int64)

y_test = torch.LongTensor(y_test)
seqlen_test = torch.LongTensor(seqlen_test)
print(f"{X_test.size() },\n{y_test.size() }\n{seqlen_test.size() }")<jupyter_output>torch.Size([1000, 1000]),
torch.Size([1000])
torch.Size([1000])

tensor dimensions <built-in method dim of Tensor object at 0x00000282562E8200>
torch.Size([200, 1000]),
torch.Size([200])
torch.Size([200])

torch.Size([200, 1000]),
torch.Size([200])
torch.Size([200])<jupyter_text>The `LSTMClassifier` classWe now define our own LSTM sequence classifier model.All PyTorch modules must subclass `nn.Module` (or one of its subclasses) and call `init` before any attribute assignment.There two methods we have to implement:- `__init__`: defines submodules. These constitute the nodes _computation graph_.- `forward` implements the forward pass of the module. This is how we map the input to the output. The way we pass the input through the module implicitly builds a directed graph of the submodules named _computation graph_.<jupyter_code>class LSTMClassifier(nn.Module):
    def __init__(self, input_size, embedding_size, hidden_size, output_size):
        super().__init__()
        self.embedding = nn.Embedding(input_size, embedding_size)
        self.lstm = nn.LSTM(
            input_size=embedding_size,
            hidden_size=hidden_size,
            num_layers=1,
            bidirectional=True,
            batch_first=True,
        )
        self.dense = nn.Linear(hidden_size * 2, output_size)
        
    # the input signature of forward changes
    def forward(self, sequences, sequence_lens):
        embedded = self.embedding(sequences)
        
        # THIS IS THE MODIFIED PART
        # returns a PackedSequence object
        packed = nn.utils.rnn.pack_padded_sequence(
            embedded,
            sequence_lens.clamp(max=1000),
            enforce_sorted=False,
            batch_first=True)
        packed_outputs, (h, c) = self.lstm(packed)
        # extract LSTM outputs (not used here)
        lstm_outputs, lens = nn.utils.rnn.pad_packed_sequence(packed_outputs)
        
        h = torch.cat((h[0], h[1]), dim=-1)
        output = self.dense(h)
        return output<jupyter_output><empty_output><jupyter_text>Instantiating the model<jupyter_code>input_size = vocab_size
embedding_size = 100
hidden_size = 64
# number of labels i.e 2
output_size = train_data.label.nunique()

model = LSTMClassifier(input_size, embedding_size, hidden_size, output_size)
model<jupyter_output><empty_output><jupyter_text>Batching<jupyter_code>class BatchedIterator:
    def __init__(self, *tensors, batch_size):
        # all tensors must have the same first dimension
        assert len(set(len(tensor) for tensor in tensors)) == 1
        self.tensors = tensors
        self.batch_size = batch_size
    
    def iterate_once(self):
        num_data = len(self.tensors[0])
        for start in range(0, num_data, self.batch_size):
            end = start + self.batch_size
            yield tuple(tensor[start:end] for tensor in self.tensors)
            
train_iter = BatchedIterator(X_train, seqlen_train, y_train, batch_size=64)
#prints number of batches
for X, seqlens, y in train_iter.iterate_once():
    print(f"{X.size()  }, {seqlens.size()  }, {y.size() }")<jupyter_output>torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([64, 1000]), torch.Size([64]), torch.Size([64])
torch.Size([40, 1000]), torch.Size([40]), torch.Size([40])<jupyter_text>Loss function and optimizerThe **loss function** or **cost function** quantifies cost of the model output differing from the expected target values.The optimizer adjusts the model's parameters in accordance with the loss.<jupyter_code>from torch import optim
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters())<jupyter_output><empty_output><jupyter_text>Sanity check on Untrained modelsTrain and val accuracy should be really bad without training.<jupyter_code># requires a large memory as we have a large vocab
logits = model(X_train, seqlen_train)
y = logits.argmax(axis=1)
accuracy = torch.sum(torch.eq(y, y_train)) / y.size(0)
print(f"Train accuracy: {accuracy:.1%}")
logits = model(X_val, seqlen_val)
y = logits.argmax(axis=1)
accuracy = torch.sum(torch.eq(y, y_val)) / y.size(0)
print(f"Val accuracy: {accuracy:.1%}")<jupyter_output>Val accuracy: 53.5%<jupyter_text>TrainingWe collect training statistics at the end of each epochs in `metrics`.<jupyter_code>from collections import defaultdict
num_epochs = 30 
batch_size = 128

metrics = defaultdict(list)
train_iter = BatchedIterator(X_train, seqlen_train, y_train, batch_size=batch_size)<jupyter_output><empty_output><jupyter_text>We train the model batch by batch and then evaluate it on the train and the dev data at the end of each epoch.Since the dataset is small, we can evaluate it the whole data in one step without batching.Note that the model should be set to **train** or **eval** mode accordingly. Stochastic steps such as dropout are disabled in **eval** mode.<jupyter_code>patience = 2
patience_counter = 0
previous_loss = 100
previous_accuracy = 0
for epoch in range(num_epochs):
   
    model.train()
    # Training loop
    for X_batch, seqlen_batch, y_batch in train_iter.iterate_once():
        y_out = model(X_batch, seqlen_batch)
        loss = criterion(y_out, y_batch)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
    with torch.no_grad():
        model.eval()  # or model.train(False)
        # Train and dev loss at the end of the epoch
        y_out = model(X_train, seqlen_train)
        train_loss = criterion(y_out, y_train).item()
        metrics['train_loss'].append(train_loss)
        labels = y_out.argmax(axis=1)
        train_accuracy = (torch.eq(y_train, labels).sum() / labels.size(0)).item()
        metrics['train_accuracy'].append(train_accuracy)

        y_out = model(X_val, seqlen_val)
        val_loss = criterion(y_out, y_val).item()
        metrics['val_loss'].append(val_loss)
        labels = y_out.argmax(axis=1)
        val_accuracy = (torch.eq(y_val, labels).sum() / labels.size(0)).item()
        metrics['val_accuracy'].append(val_accuracy)
        
        if val_accuracy <= previous_accuracy and val_loss <= previous_loss:
            patience_counter += 1
            
        previous_accuracy = val_accuracy
        previous_loss = val_loss
            
    
    print(f"{epoch} -- {train_loss:.3f} - {train_accuracy:.1%} - {val_loss:.3f} - {val_accuracy:.1%}")
    
#     if patience_counter == patience:
#             break<jupyter_output>0 -- 0.693 - 49.2% - 0.691 - 53.5%
1 -- 0.691 - 53.0% - 0.696 - 49.0%
2 -- 0.689 - 53.7% - 0.693 - 49.5%
3 -- 0.688 - 54.0% - 0.691 - 49.5%
4 -- 0.686 - 54.3% - 0.693 - 49.5%
5 -- 0.685 - 54.2% - 0.691 - 49.0%
6 -- 0.683 - 54.3% - 0.692 - 49.0%
7 -- 0.681 - 54.3% - 0.691 - 49.0%
8 -- 0.679 - 54.5% - 0.691 - 49.0%
9 -- 0.676 - 55.1% - 0.692 - 49.0%
10 -- 0.673 - 55.1% - 0.692 - 49.0%
11 -- 0.669 - 55.2% - 0.693 - 49.0%
12 -- 0.665 - 55.5% - 0.694 - 49.0%
13 -- 0.660 - 55.6% - 0.697 - 48.5%
14 -- 0.654 - 55.6% - 0.703 - 47.5%
15 -- 0.648 - 55.8% - 0.713 - 47.5%
16 -- 0.646 - 55.7% - 0.725 - 48.0%
17 -- 0.645 - 55.8% - 0.724 - 47.5%
18 -- 0.642 - 55.8% - 0.727 - 47.5%
19 -- 0.640 - 55.8% - 0.731 - 47.0%
20 -- 0.639 - 55.8% - 0.736 - 47.5%
21 -- 0.638 - 55.8% - 0.755 - 47.5%
22 -- 0.637 - 55.8% - 0.767 - 47.0%
23 -- 0.637 - 55.8% - 0.755 - 47.5%
24 -- 0.637 - 55.8% - 0.761 - 48.0%
25 -- 0.637 - 55.8% - 0.787 - 47.0%
26 -- 0.636 - 55.8% - 0.792 - 47.0%
27 -- 0.636 - 55.8% - 0.795 - 47.0%
28[...]<jupyter_text>Evaluation Training curves<jupyter_code>import matplotlib.pyplot as plt
import seaborn as sns

fig, ax = plt.subplots(1, 2, figsize=(16, 4))

sns.lineplot(data=metrics['train_loss'], ax=ax[0], label='train loss')
sns.lineplot(data=metrics['val_loss'], ax=ax[0], label='val loss')

sns.lineplot(data=metrics['train_accuracy'], ax=ax[1], label='train acc')
sns.lineplot(data=metrics['val_accuracy'], ax=ax[1], label='val acc')<jupyter_output><empty_output><jupyter_text>Test accuracy<jupyter_code>from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix
logits = model(X_test, seqlen_test)
test_prediction = logits.argmax(axis=1)
test_accuracy = torch.sum(torch.eq(test_prediction, y_test)) / y_test.size(0)
print(f"Test accuracy: {test_accuracy:.1%}")
logits = model(X_test,seqlen_test)

test_pred = np.argmax(logits.detach().numpy(),axis =1)
test_precision = precision_score(y_test,test_pred,average = "macro")
print(f"Test precision : {test_precision:.1%}")
r1 = recall_score(y_test,test_pred,average = "macro")
print(f"Recall score :{r1: .1%}")
mat = confusion_matrix(y_test,test_pred)
mat<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1447328931572629, 'fraction_numerical': 0.05889855942376951, 'mean_word_length': 4.071917808219178, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '44155', 'license': 'MIT', 'n_tokens_mistral': 5398, 'n_tokens_neox': 4858, 'n_words': 1058, 'path': 'LSTM_kaggle.ipynb', 'repo_name': 'AirinB/Sentiment-Analysis', 'size': '13328'}