<jupyter_start><jupyter_text>BERT Modeling The goal for this analysis is to predict if a review rates the movie positively or negatively. Inside this dataset there are 25,000 labeled movies reviews for training, 50,000 unlabeled reviews for training, and 25,000 reviews for testing.- IMDB movie reviews dataset- http://ai.stanford.edu/~amaas/data/sentiment- https://www.kaggle.com/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews- Contains 25000 positive and 25000 negative reviews- Contains at most reviews per movie- At least 7 stars out of 10 $\rightarrow$ positive (label = 1)- At most 4 stars out of 10 $\rightarrow$ negative (label = 0)> **Here, we use Bert**   Dependencies<jupyter_code>import torch
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score, confusion_matrix
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
! pip install transformers
from transformers import BertTokenizer, BertForSequenceClassification
from transformers import AdamW, get_linear_schedule_with_warmup
# for importing data to colab
from google.colab import drive 
drive.mount('/content/drive')

movies = pd.read_csv('/content/drive/My Drive/Colab Notebooks/imdb_data.csv')
movies.sample(7)
# Cateogrize positive and negative as 1 and 0 respectively
label_encoder = LabelEncoder()
movies['sentiment'] = label_encoder.fit_transform(movies['sentiment'])
movies.head()<jupyter_output><empty_output><jupyter_text>Split the Data<jupyter_code>X_train, X_val, y_train, y_val = train_test_split(movies.index, movies['sentiment'], test_size=0.2, random_state=7)
X_train[:5]
movies['data_type'] = ['not_set']*movies.shape[0]
movies.loc[X_train, 'data_type'] = 'train'
movies.loc[X_val, 'data_type'] = 'val'
movies.groupby(['sentiment', 'data_type']).count()<jupyter_output><empty_output><jupyter_text>Tokenize and Encode the data<jupyter_code>tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
encoded_data_train = tokenizer.batch_encode_plus(
    movies[movies.data_type=='train'].review.values, 
    add_special_tokens=True, 
    return_attention_mask=True, 
    pad_to_max_length=True, 
    max_length=256, 
    return_tensors='pt'
)

encoded_data_val = tokenizer.batch_encode_plus(
    movies[movies.data_type=='val'].review.values, 
    add_special_tokens=True, 
    return_attention_mask=True, 
    pad_to_max_length=True, 
    max_length=256, 
    return_tensors='pt'
)


input_ids_train = encoded_data_train['input_ids']
attention_masks_train = encoded_data_train['attention_mask']
labels_train = torch.tensor(movies[movies.data_type=='train'].sentiment.values)

input_ids_val = encoded_data_val['input_ids']
attention_masks_val = encoded_data_val['attention_mask']
labels_val = torch.tensor(movies[movies.data_type=='val'].sentiment.values)
dataset_train = TensorDataset(input_ids_train, attention_masks_train, labels_train)
dataset_val = TensorDataset(input_ids_val, attention_masks_val, labels_val)
len(dataset_train)
len(dataset_val)<jupyter_output><empty_output><jupyter_text>BERT Pretarined Model<jupyter_code>model = BertForSequenceClassification.from_pretrained("bert-base-uncased",
                                                      num_labels=2, # positive and negative
                                                      output_attentions=False,
                                                      output_hidden_states=False)<jupyter_output><empty_output><jupyter_text>DataLoaders<jupyter_code>batch_size = 32

dataloader_train = DataLoader(dataset_train, 
                              sampler=RandomSampler(dataset_train), 
                              batch_size=batch_size)

dataloader_validation = DataLoader(dataset_val, 
                                   sampler=SequentialSampler(dataset_val), 
                                   batch_size=batch_size)<jupyter_output><empty_output><jupyter_text>Optimizer and Scheduler<jupyter_code>optimizer = AdamW(model.parameters(),
                  lr=1e-5, 
                  eps=1e-8)
epochs = 3

scheduler = get_linear_schedule_with_warmup(optimizer, 
                                            num_warmup_steps=0,
                                            num_training_steps=len(dataloader_train)*epochs)<jupyter_output><empty_output><jupyter_text>Creating our Training Loop<jupyter_code>import random

seed_val = 7
random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)
train_on_gpu=torch.cuda.is_available()
device = torch.device('cuda' if train_on_gpu else 'cpu')

model.to(device)
# If there's a GPU available...
if train_on_gpu:    
    device = torch.device("cuda")

    print('There are %d GPU(s) available.' % torch.cuda.device_count())

    print('Triaining on GPU:', torch.cuda.get_device_name(0))

else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")
def evaluate(dataloader_val):

    model.eval()
    
    loss_val_total = 0
    predictions, true_vals = [], []
    
    for batch in dataloader_val:
        
        batch = tuple(b.to(device) for b in batch)
        
        inputs = {'input_ids':      batch[0],
                  'attention_mask': batch[1],
                  'labels':         batch[2],
                 }

        with torch.no_grad():        
            outputs = model(**inputs)
            
        loss = outputs[0]
        logits = outputs[1]
        loss_val_total += loss.item()

        logits = logits.detach().cpu().numpy()
        label_ids = inputs['labels'].cpu().numpy()
        predictions.append(logits)
        true_vals.append(label_ids)
    
    loss_val_avg = loss_val_total/len(dataloader_val) 
    
    predictions = np.concatenate(predictions, axis=0)
    true_vals = np.concatenate(true_vals, axis=0)
            
    return loss_val_avg, predictions, true_vals
# Training, Takes a lot of time. Use GPU preferably...
for epoch in tqdm(range(1, epochs+1)):
    
    model.train()
    
    loss_train_total = 0

    progress_bar = tqdm(dataloader_train, desc='Epoch {:1d}'.format(epoch), leave=False, disable=False)
    for batch in progress_bar:

        model.zero_grad()
        
        batch = tuple(b.to(device) for b in batch)
        
        inputs = {'input_ids':      batch[0],
                  'attention_mask': batch[1],
                  'labels':         batch[2],
                 }       

        outputs = model(**inputs)
        
        loss = outputs[0]
        loss_train_total += loss.item()
        loss.backward()

        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

        optimizer.step()
        scheduler.step()
        
        progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item()/len(batch))})
         
        
    torch.save(model.state_dict(), f'finetuned_BERT_epoch_{epoch}.model')
        
    tqdm.write(f'\nEpoch {epoch}')
    
    loss_train_avg = loss_train_total/len(dataloader_train)            
    tqdm.write(f'Training loss: {loss_train_avg}')
    
    val_loss, predictions, true_vals = evaluate(dataloader_validation)
    tqdm.write(f'Validation loss: {val_loss}')
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",
                                                      num_labels=2,
                                                      output_attentions=False,
                                                      output_hidden_states=False)

model.to(device)
if(train_on_gpu):
    model.load_state_dict(torch.load('finetuned_BERT_epoch_1.model', map_location=torch.device('cuda')))
else:
    model.load_state_dict(torch.load('finetuned_BERT_epoch_1.model', map_location=torch.device('cpu')))

_, predictions, true_vals = evaluate(dataloader_validation)
def accuracy_per_class(preds, labels):
    possible_labels = movies.sentiment.unique()
    label_dict = {}
    for index, possible_label in enumerate(possible_labels):
        label_dict[possible_label] = index
    label_dict_inverse = {v: k for k, v in label_dict.items()}
    
    preds_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    correct = 0
    for label in np.unique(labels_flat):
        y_preds = preds_flat[labels_flat==label]
        y_true = labels_flat[labels_flat==label]
        print(f'Class: {label_dict_inverse[label]}')
        print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}\n')
        correct += len(y_preds[y_preds==label])
    print("Total Validation Set Accuracy: {:.3f}".format(correct/X_val.shape[0]*100))
    
accuracy_per_class(predictions, true_vals)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10585148853655754, 'fraction_numerical': 0.010379833466408122, 'mean_word_length': 3.191204588910134, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '77606', 'license': 'MIT', 'n_tokens_mistral': 2856, 'n_tokens_neox': 2745, 'n_words': 487, 'path': 'Bert_model_predictions.ipynb', 'repo_name': 'SuryaPradeepM/Comprehensive-Sentiment-Analysis-of-Movie-Reviews-', 'size': '8767'}