<jupyter_start><jupyter_text>BiLSTM with Attention<jupyter_code>class LSTMAttention(torch.nn.Module):
    def __init__(self, label_size, hidden_dim, batch_size, embedding_matrix,
                keep_dropout=.5, lstm_layers=1):

        super(LSTMAttention, self).__init__()
        self.hidden_dim = hidden_dim
        self.batch_size = batch_size
        self.use_gpu = torch.cuda.is_available()
        self.embedding_dim = embedding_matrix.shape[1]
        self.word_embeddings = nn.Embedding.from_pretrained(embedding_matrix, freeze=False)
        self.num_layers = lstm_layers
        self.dropout = keep_dropout
        self.bilstm = nn.LSTM(self.embedding_dim, hidden_dim // 2, batch_first=True, num_layers=self.num_layers,
                              dropout=self.dropout, bidirectional=True)
        self.hidden2label = nn.Linear(hidden_dim, label_size)
        self.hidden = self.init_hidden()
        ##self.mean = opt.__dict__.get("lstm_mean",True) -- no use in the class
        self.attn_fc = torch.nn.Linear(self.embedding_dim, 1)

    def init_hidden(self, batch_size=None):
        """
        initialize hidden state and cell state
        """
        if batch_size is None:
            batch_size= self.batch_size
        
        if self.use_gpu:
            h0 = Variable(torch.zeros(2*self.num_layers, batch_size, self.hidden_dim // 2).cuda())
            c0 = Variable(torch.zeros(2*self.num_layers, batch_size, self.hidden_dim // 2).cuda())
        else:
            h0 = Variable(torch.zeros(2*self.num_layers, batch_size, self.hidden_dim // 2))
            c0 = Variable(torch.zeros(2*self.num_layers, batch_size, self.hidden_dim // 2))
        return (h0, c0)


    def attention(self, rnn_out, state):
        merged_state = torch.cat([s for s in state],1)
        merged_state = merged_state.squeeze(0).unsqueeze(2)
        # (batch, seq_len, cell_size) * (batch, cell_size, 1) = (batch, seq_len, 1)
        weights = torch.bmm(rnn_out, merged_state)
        weights = torch.nn.functional.softmax(weights.squeeze(2)).unsqueeze(2)
        # (batch, cell_size, seq_len) * (batch, seq_len, 1) = (batch, cell_size, 1)
        return torch.bmm(torch.transpose(rnn_out, 1, 2), weights).squeeze(2)
    

    def forward(self, X):
        embedded = self.word_embeddings(X)
        hidden= self.init_hidden(X.size()[0]) 
        rnn_out, hidden = self.bilstm(embedded, hidden)
        h_n, c_n = hidden
        attn_out = self.attention(rnn_out, h_n)
        logits = self.hidden2label(attn_out)
        return logits
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
import time,datetime
from sklearn.metrics import classification_report
import random
from torch.optim.lr_scheduler import StepLR
import pdb
# Function to calculate the accuracy of our predictions vs labels
def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)

def format_time(elapsed):
    '''
    Takes a time in seconds and returns a string hh:mm:ss
    '''
    # Round to the nearest second.
    elapsed_rounded = int(round((elapsed)))
    
    # Format as hh:mm:ss
    return str(datetime.timedelta(seconds=elapsed_rounded))

# If there's a GPU available...
if torch.cuda.is_available():    

    # Tell PyTorch to use the GPU.    
    device = torch.device("cuda")

    print('There are %d GPU(s) available.' % torch.cuda.device_count())

    print('We will use the GPU:', torch.cuda.get_device_name(0))

# If not...
else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")

### evaluate the performance of current model
def evaluate_model(clf_model, validation_dataloader, save_dir):
    """
    :params[in]: clf_model, the pre-trained classifier
    :params[in]: validation_dataloader, the validation dataset
    :params[in]: save_dir, the directory name to save the fine-tuned model
    
    """
    t0 = time.time()
    # Put the model in evaluation mode--the dropout layers behave differently
    # during evaluation.
    clf_model.eval()
    # Tracking variables 
    eval_loss, eval_accuracy = 0, 0
    nb_eval_steps, nb_eval_examples = 0, 0
    true_labels,pred_labels=[],[]
    # Evaluate data for one epoch
    for batch in validation_dataloader:
        
        # Add batch to GPU
        batch = tuple(t.to(device) for t in batch)
        
        # Unpack the inputs from our dataloader
        b_input_ids, b_labels = batch
        
        # Telling the model not to compute or store gradients, saving memory and
        # speeding up validation
        with torch.no_grad():        
            # Forward pass, calculate logit predictions.
            # This will return the logits rather than the loss because we have
            # not provided labels.
            # token_type_ids is the same as the "segment ids", which 
            # differentiates sentence 1 and 2 in 2-sentence tasks.
            # The documentation for this `model` function is here: 
            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification
            outputs = clf_model(b_input_ids)
        
        # Get the "logits" output by the model. The "logits" are the output
        # values prior to applying an activation function like the softmax.
        logits = outputs

        # Move logits and labels to CPU
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        # Calculate the accuracy for this batch of test sentences.
        tmp_eval_accuracy = flat_accuracy(logits, label_ids)
        ## pred_labels/true_labels in a batch flatten
        pred_flat = np.argmax(logits, axis=1).flatten()
        true_flat = label_ids.flatten()

        # true labels and predicted labels
        true_labels += true_flat.tolist()
        pred_labels += pred_flat.tolist()
        # Accumulate the total accuracy.
        eval_accuracy += tmp_eval_accuracy
        # Track the number of batches
        nb_eval_steps += 1
    ## pdb check
    #pdb.set_trace()
    # Report the final accuracy for this validation run
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    #clf_model.save_pretrained(save_dir)  ## save model
    print(classification_report(true_labels, pred_labels,digits=3))
    print(classification_report(true_labels, pred_labels,digits=3),
      file=open(save_dir+'/result.txt','w'))
    print("  Accuracy: {0:.3f}".format(eval_accuracy/nb_eval_steps),
          file=open(save_dir+'/result.txt','w'))

def train_eval(clf_model, train_dataloader, validation_dataloader, base_dir,
               lr, epochs=4, eval_every_num_iters=40, seed_val = 42, weights= None):
    """train and evaluate a deep learning model
    :params[in]: clf_model, a classifier
    :params[in]: train_dataloader, training data
    :params[in]: validation_dataloader, validation data
    :params[in]: base_dir, output directory to create the directory to save results
    :params[in]: lr, the learning rate
    :params[in]: epochs, the number of training epochs
    :params[in]: eval_every_num_iters, the number of iterations to evaluate
    :params[in]: seed_val, set a random seed
    """
    optimizer = torch.optim.Adam(clf_model.parameters(),
                      lr = lr)
    ## cross entropy loss
    criterion = nn.CrossEntropyLoss()

    # Total number of training steps is number of batches * number of epochs.
    total_steps = len(train_dataloader) * epochs
    # Create the learning rate scheduler. # gamma = decaying factor
    scheduler = StepLR(optimizer, step_size=2, gamma=0.5)

    # see if weights is None:
    if weights != None:
        weights = torch.FloatTensor(weights)
    # Set the seed value all over the place to make this reproducible.
    random.seed(seed_val)
    np.random.seed(seed_val)
    torch.manual_seed(seed_val)
    torch.cuda.manual_seed_all(seed_val)
    
    # Store the average loss after each epoch so we can plot them.
    loss_values = []
    
    # For each epoch...
    for epoch_i in range(0, epochs):
        
        # ========================================
        #               Training
        # ========================================
        print("")
        print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
        print('Training...')
        
        # Measure how long the training epoch takes.
        t0 = time.time()
    
        # Reset the total loss for this epoch.
        total_loss = 0
        ## print the learning rate
        for param_group in optimizer.param_groups:
            print(' learning rate is: ', param_group['lr'])
        
        # Put the model into training mode. Don't be mislead--the call to 
        # `train` just changes the *mode*, it doesn't *perform* the training.
        # `dropout` and `batchnorm` layers behave differently during training
        # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)
    
        # For each batch of training data...
        for iters, batch in enumerate(train_dataloader):
            clf_model.train()  ## model training mode
            # Unpack this training batch from our dataloader. 
            #
            # As we unpack the batch, we'll also copy each tensor to the GPU using the 
            # `to` method.
            #
            # `batch` contains three pytorch tensors:
            #   [0]: input ids 
            #   [1]: attention masks
            #   [2]: labels 
            b_input_ids = batch[0].to(device)
            b_labels = batch[1].to(device)
    
            # Always clear any previously calculated gradients before performing a
            # backward pass. PyTorch doesn't do this automatically because 
            # accumulating the gradients is "convenient while training RNNs". 
            # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)
            clf_model.zero_grad()        
    
            # Perform a forward pass (evaluate the model on this training batch).
            # This will return the loss (rather than the model output) because we
            # have provided the `labels`.
            # The documentation for this `model` function is here: 
            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification
            outputs = clf_model(b_input_ids)
             
            # The call to `model` always returns a tuple, so we need to pull the 
            # loss value out of the tuple.
            loss = criterion(outputs, b_labels)
    
            # Accumulate the training loss over all of the batches so that we can
            # calculate the average loss at the end. `loss` is a Tensor containing a
            # single value; the `.item()` function just returns the Python value 
            # from the tensor.
            total_loss += loss.item()
    
            # Perform a backward pass to calculate the gradients.
            loss.backward()
    
            # Clip the norm of the gradients to 1.0.
            # This is to help prevent the "exploding gradients" problem.
            # torch.nn.utils.clip_grad_norm_(clf_model.parameters(), 1.0)
            # Update parameters and take a step using the computed gradient.
            # The optimizer dictates the "update rule"--how the parameters are
            # modified based on their gradients, the learning rate, etc.
            optimizer.step()    
            # eveluate the performance after some iterations
            if iters % eval_every_num_iters == 0 and not iters == 0:
                # Calculate elapsed time in minutes.
                elapsed = format_time(time.time() - t0)
                # Report progress.
                print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(iters, len(train_dataloader), elapsed))
                tmp_dir = base_dir+'/epoch'+str(epoch_i+1)+'iteration'+str(iters)
                ## save pretrained model
                evaluate_model(clf_model, validation_dataloader, tmp_dir)
        # Update the learning rate each epoch
        scheduler.step()
        # Calculate the average loss over the training data.
        avg_train_loss = total_loss / len(train_dataloader)            
        #pdb.set_trace()
        # Store the loss value for plotting the learning curve.
        loss_values.append(avg_train_loss)
        # save the data after epochs
        tmp_dir = base_dir+'/epoch'+str(epoch_i+1)+'_done'
        ## save pretrained model
        evaluate_model(clf_model, validation_dataloader, tmp_dir)

from torch.autograd import Variable
    import os
    for seed in [42,52, 62, 72, 82]:
        batch_size = 8
        train_ids,test_ids,wd_ind,emb_ind=loadData_Tokenizer(X_train, X_test, MAX_NB_WORDS=10000,
                       MAX_SEQUENCE_LENGTH=256, w2v_file=w2v_file)
        embedding_matrix=init_embed(wd_ind, emb_ind, emb_dim=100)
        ## initialize a classifier
        clf_model=LSTMAttention(label_size=3, hidden_dim=60, batch_size=batch_size, lstm_layers=1, 
                keep_dropout=.5, embedding_matrix=torch.FloatTensor(embedding_matrix))
        # Create the DataLoader for our training set.
        train_data = TensorDataset(torch.LongTensor(train_ids), torch.tensor(y_train))
        train_sampler = RandomSampler(train_data)
        train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
        # Create the DataLoader for our validation set.
        validation_data = TensorDataset(torch.LongTensor(test_ids), torch.tensor(y_test))
        validation_sampler = SequentialSampler(validation_data)
        validation_dataloader = DataLoader(validation_data, sampler=validation_sampler,
                                           batch_size=batch_size)
        base_dir = 'textLSTM_att/LSTM_att_seed'+str(seed)
        train_eval(clf_model, train_dataloader, validation_dataloader, base_dir, \
            lr=1.0e-2, epochs=5, eval_every_num_iters=160, seed_val = seed)

##text cnn
res = np.array([[0.732,0.731,0.734,0.858],
               [0.761,0.762,0.761,0.875],
                [0.773,0.776,0.771,0.881],
                [0.779,0.772,0.789,0.872],
                [0.760,0.749,0.790,0.847]
               ])
np.mean(res, axis=0)
np.std(res, axis=0)
##BERT
res = np.array([[0.760,0.764,0.757,0.881],
               [0.790,0.787,0.794,0.889],
                [0.730,0.734,0.727,0.872],
                [0.763,0.762,0.765,0.881]
               ])
np.mean(res, axis=0)
np.std(res, axis=0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1032139702384997, 'fraction_numerical': 0.019297411157165184, 'mean_word_length': 2.500118906064209, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '37339', 'license': 'Apache-2.0', 'n_tokens_mistral': 4682, 'n_tokens_neox': 4429, 'n_words': 1160, 'path': 'python_notebooks/emergency_project_lstm_attention.ipynb', 'repo_name': 'shijing001/Text_classifier', 'size': '14717'}