<jupyter_start><jupyter_text>NER: BiLSTM-LSTM ModelUse LSTM as the decoder. For the time being consider no batching. Later implement batching as well.<jupyter_code>import logging
import os

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

from tqdm import trange
from tqdm import tqdm_notebook as tqdm
import random

import utils
from model.data_loader import DataLoader
from evaluate import evaluate, f_score_simple
data_dir = 'data/coNLL/eng/'
model_dir = 'experiments/coNLL/lstm_model/'
# data_dir = 'data/kaggle/'
# model_dir = 'experiments/kaggle/lstm_model/'
path_glove = '../sequence_tagging/data/glove.6B/glove.6B.100d.txt'

json_path = os.path.join(model_dir, 'params.json')
params = utils.Params(json_path)
# use GPU if available
params.cuda = torch.cuda.is_available()
params.use_glove = True
params.dropout = 0.1
params.dict
# load data
data_loader = DataLoader(data_dir, params, path_glove)
data = data_loader.load_data(['train', 'val', 'test'])
train_data = data['train']
val_data = data['val']
test_data = data['test']

# specify the train and val dataset sizes
params.train_size = train_data['size']
params.val_size = val_data['size']
params.test_size = test_data['size']

params.pad_tag_ind = data_loader.tag_map[params.pad_tag]
params.batch_size = 1
SOS_token = -1<jupyter_output>14041 14041
3250 3250
3453 3453<jupyter_text>Model<jupyter_code>class EncoderRNN(nn.Module):
    
    def __init__(self, params, embedding):
        '''embedding is the glove embedding loaded in the data_loader.py'''
        super(EncoderRNN, self).__init__()
        
        # the embedding takes as input the vocab_size and the embedding_dim
        self.embedding = nn.Embedding(params.vocab_size, params.embedding_dim)
        # copy pretrained embedding
        self.embedding.weight.data.copy_(torch.from_numpy(embedding))
        
        self.dropout = nn.Dropout(0.1)
        
        # the LSTM takes as input the size of its input (embedding_dim), its hidden size
        # for more details on how to use it, check out the documentation
        self.encoder = nn.GRU(params.embedding_dim, params.lstm_hidden_dim, batch_first=True)
        
    def forward(self, s):   
        # apply the embedding layer that maps each token to its embedding
        s = self.embedding(s)            # dim: seq_len x batch_size x embedding_dim
        s = self.dropout(s)

        # run the LSTM along the sentences of length seq_len
        output, hidden = self.encoder(s)    # dim: seq_len x batch_size x lstm_hidden_dim

        return output, hidden


    
class DecoderRNN(nn.Module):
        
    def __init__(self, params):
        
        super(DecoderRNN, self).__init__()
        
        self.hidden_size = params.lstm_hidden_dim
        self.tag_size = params.number_of_tags # SOS is represented as all zero vector
        self.decoder = nn.GRU(self.tag_size + self.hidden_size, self.hidden_size, batch_first=True)
        
        # the fully connected layer transforms the output to give the final output layer
        self.fc = nn.Linear(self.hidden_size, self.tag_size)
        
    
    def forward(self, prev_tag, hidden, encoder_output):
        '''prev_tag: is a scalar
           hidden: is the previous hidden vector: as we are now processing one sentence word by word
           encoder_output: encoder's  corresponding output in the input sentence
           
           output: 1 x tag_size
        '''
        tag = torch.zeros(1, self.tag_size).cuda()
        if prev_tag >= 0:
            tag[0][prev_tag] = 1 # 1 hot encoding
            
        encoder_output = encoder_output.view(1, -1)
        output = torch.cat((tag, encoder_output), dim=1)
        output = output.unsqueeze(1)
        
        # apply one cell of GRU
        output, hidden = self.decoder(output, hidden)  
        
        # apply the fully connected layer and obtain the output (before softmax) for each token
        output = self.fc(output[0])                   # dim: batch_size*seq_len x num_tags

        # apply log softmax on each token's output 
        output = F.log_softmax(output, dim=1)   # dim: batch_size*seq_len x num_tags
        
        return output, hidden
# Set the logger
utils.set_logger(os.path.join(model_dir, 'train.log'))<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>teacher_forcing_ratio = 0.5

def train(data_iterator, encoder, decoder, encoder_optimizer, decoder_optimizer, num_steps):
    
    # set model to training mode
    encoder.train()
    decoder.train()

    # Running average object for loss, accuracy 
    loss_avg = utils.RunningAverage()
    acc_avg = utils.RunningAverage()
    other_ind = data_loader.tag_map['O']
    recall = utils.RunningAverage()
    precision = utils.RunningAverage()
    
    t = trange(num_steps) 
    
    for i in t:

        encoder_optimizer.zero_grad()
        decoder_optimizer.zero_grad()

        train_batch, labels_batch, _ = next(data_iterator) 

        encoder_outputs, encoder_hidden = encoder(train_batch)
        decoder_hidden = encoder_hidden
        loss = 0
        prev_token = SOS_token # -1
        predicted_tokens = torch.zeros_like(labels_batch)
        
        use_teacher_forcing = False #True if random.random() < teacher_forcing_ratio else False

        if use_teacher_forcing:
            # Teacher forcing: Feed the target as the next input
            for di in range(labels_batch.shape[1]):
                decoder_output, decoder_hidden = decoder(prev_token, decoder_hidden, encoder_outputs[:,di,:])
                prev_token = labels_batch[:, di].item() # Teacher forcing
                _, predicted_tokens[0][di] = torch.max(decoder_output.view(-1), dim=0)
                loss += criterion(decoder_output, labels_batch[:, di])

        else:
            # Without teacher forcing: use its own predictions as the next input
            for di in range(labels_batch.shape[1]):
                decoder_output, decoder_hidden = decoder(prev_token, decoder_hidden, encoder_outputs[:,di,:])
                topv, topi = torch.max(decoder_output.view(-1), dim=0)
                prev_token = topi.item()
                predicted_tokens[0][di] = topi
                loss += criterion(decoder_output, labels_batch[:,di])
        
        # find accuracy of prediction
        correct = (predicted_tokens == labels_batch).view(-1).cpu().numpy()
        acc_avg.update(sum(correct), len(correct))
        
        # find accuracy (recall) for only named entities other than O
        ne_labels = labels_batch.view(-1).cpu().numpy() != other_ind
        recall.update(sum(correct & ne_labels), sum(ne_labels))
        
        # find precision of ne predictions
        ne_preds = predicted_tokens.view(-1).cpu().numpy() != other_ind
        precision.update(sum(correct & ne_preds), sum(ne_preds))
        
        # find f-score
        if (recall() + precision()) == 0:
            f_score = 0
        else:
            f_score = 2*recall()*precision() / (recall() + precision())
        
        # update the average loss
        loss_avg.update(loss.item())
        
        
        t.set_postfix(loss='{:05.3f}'.format(loss_avg()), acc='{:05.3f}'.format(acc_avg()), f1='{:05.3f}'.format(f_score))
        
        loss.backward()
        encoder_optimizer.step()
        decoder_optimizer.step()
def evaluate(data_iterator, encoder, decoder, num_steps):
    
    logging.info('Evaluating:')
    # set model to eval mode
    encoder.eval()
    decoder.eval()

    # Running average object for loss, accuracy and ne_accuracy
    # ne_accuracy: omit the 'O' tag for computing accuracy
    loss_avg = utils.RunningAverage()
    acc_avg = utils.RunningAverage()
    other_ind = data_loader.tag_map['O']
    recall = utils.RunningAverage()
    precision = utils.RunningAverage()
    
    t = trange(num_steps) 
    
    for i in t:

        test_batch, labels_batch, _ = next(data_iterator) 

        encoder_outputs, encoder_hidden = encoder(test_batch)
        decoder_hidden = encoder_hidden
        loss = 0
        prev_token = SOS_token # -1
        predicted_tokens = torch.zeros_like(labels_batch)

        for di in range(labels_batch.shape[1]):
            decoder_output, decoder_hidden = decoder(prev_token, decoder_hidden, encoder_outputs[:,di,:])
            topv, topi = torch.max(decoder_output.view(-1), dim=0)
            prev_token = topi.item()
            predicted_tokens[0][di] = topi
            loss += criterion(decoder_output, labels_batch[:,di])
        
        # find accuracy of prediction
        correct = (predicted_tokens == labels_batch).view(-1).cpu().numpy()
        acc_avg.update(sum(correct), len(correct))
        
        # find accuracy (recall) for only named entities other than O
        ne_labels = labels_batch.view(-1).cpu().numpy() != other_ind
        recall.update(sum(correct & ne_labels), sum(ne_labels))
        
        # find precision of ne predictions
        ne_preds = predicted_tokens.view(-1).cpu().numpy() != other_ind
        precision.update(sum(correct & ne_preds), sum(ne_preds))
        
        # find f-score
        if (recall() + precision()) == 0:
            f_score = 0
        else:
            f_score = 2*recall()*precision() / (recall() + precision())
        
        # update the average loss
        loss_avg.update(loss.item())
        
        t.set_postfix(loss='{:05.3f}'.format(loss_avg()), acc='{:05.3f}'.format(acc_avg()), f1='{:05.3f}'.format(f_score))
    
    return f_score
# Define the model and optimizer
encoder1 = EncoderRNN(params, data_loader.embedding).cuda()
decoder1 = DecoderRNN(params).cuda()

criterion = nn.NLLLoss()

#encoder_optimizer = optim.SGD(encoder1.parameters(), lr=params.learning_rate, momentum=0.9)
#decoder_optimizer = optim.SGD(decoder1.parameters(), lr=params.learning_rate, momentum=0.9)
encoder_optimizer = optim.Adam(encoder1.parameters(), lr=params.learning_rate)
decoder_optimizer = optim.Adam(decoder1.parameters(), lr=params.learning_rate)
# params.num_epochs = 10
best_val_acc = 0.0

for epoch in range(params.num_epochs):
    # Run one epoch
    logging.info("Epoch {}/{}".format(epoch + 1, params.num_epochs))

    # compute number of batches in one epoch (one full pass over the training set)
    # num_steps = params.train_size
    train_data_iterator = data_loader.data_iterator(train_data, params, shuffle=True)
    train(train_data_iterator, encoder1, decoder1, encoder_optimizer, decoder_optimizer, params.train_size)

    # Evaluate for one epoch on validation set
    # num_steps = params.val_size 
    val_data_iterator = data_loader.data_iterator(val_data, params, shuffle=False)
    val_acc = evaluate(val_data_iterator, encoder1, decoder1, params.val_size)
    val_metrics = {"f1": val_acc}
    
    is_best = val_acc >= best_val_acc

    # Save weights
    state_encoder = {'epoch': epoch + 1, 'state_dict': encoder1.state_dict(), 'optim_dict' : encoder_optimizer.state_dict()}
    utils.save_checkpoint(state_encoder, is_best=is_best, checkpoint=model_dir, extra='encoder_')
    
    state_decoder = {'epoch': epoch + 1, 'state_dict': decoder1.state_dict(), 'optim_dict' : decoder_optimizer.state_dict()}
    utils.save_checkpoint(state_decoder, is_best=is_best, checkpoint=model_dir, extra='decoder_')

    # If best_eval, best_save_path        
    if is_best:
        logging.info("- Found new best accuracy")
        best_val_acc = val_acc

        # Save best val metrics in a json file in the model directory
        best_json_path = os.path.join(model_dir, "f1_score_best_weights.json")
        utils.save_dict_to_json(val_metrics, best_json_path)

    # Save latest val metrics in a json file in the model directory
    last_json_path = os.path.join(model_dir, "f1_score_last_weights.json")
    utils.save_dict_to_json(val_metrics, last_json_path)<jupyter_output>Epoch 1/15
100%|██████████| 14041/14041 [08:00<00:00, 29.22it/s, acc=0.956, f1=0.826, loss=2.109]
Evaluating:
100%|██████████| 3250/3250 [01:05<00:00, 49.37it/s, acc=0.968, f1=0.872, loss=1.650]
- Found new best accuracy
Epoch 2/15
100%|██████████| 14041/14041 [08:04<00:00, 28.99it/s, acc=0.982, f1=0.923, loss=0.845]
Evaluating:
100%|██████████| 3250/3250 [01:07<00:00, 48.04it/s, acc=0.972, f1=0.888, loss=1.534]
- Found new best accuracy
Epoch 3/15
100%|██████████| 14041/14041 [08:02<00:00, 29.09it/s, acc=0.989, f1=0.952, loss=0.510]
Evaluating:
100%|██████████| 3250/3250 [01:06<00:00, 49.23it/s, acc=0.971, f1=0.883, loss=1.658]
Epoch 4/15
100%|██████████| 14041/14041 [08:02<00:00, 29.12it/s, acc=0.993, f1=0.968, loss=0.342]
Evaluating:
100%|██████████| 3250/3250 [01:06<00:00, 48.99it/s, acc=0.971, f1=0.884, loss=1.805]
Epoch 5/15
100%|██████████| 14041/14041 [06:25<00:00, 36.42it/s, acc=0.995, f1=0.978, loss=0.233]
Evaluating:
100%|██████████| 3250/3250 [00:34<00:00, 92.93it/s, acc=0.[...]<jupyter_text>Test<jupyter_code>test_data_iterator = data_loader.data_iterator(test_data, params, shuffle=False)
evaluate(test_data_iterator, encoder1, decoder1, params.test_size)
restore_file = 'encoder_best'
# Reload weights from the saved file
r = utils.load_checkpoint(os.path.join(model_dir, restore_file + '.pth.tar'), encoder1)

restore_file = 'decoder_best'
# Reload weights from the saved file
r = utils.load_checkpoint(os.path.join(model_dir, restore_file + '.pth.tar'), decoder1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11979748343384707, 'fraction_numerical': 0.03737621919440101, 'mean_word_length': 3.1675457648153893, 'pattern_counts': {'":': 1, '<': 26, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '516092', 'license': 'MIT', 'n_tokens_mistral': 4839, 'n_tokens_neox': 4407, 'n_words': 997, 'path': 'NER-LSTM.ipynb', 'repo_name': 'bnjasim/named-entity-recognition', 'size': '13431'}