<jupyter_start><jupyter_text><jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.datasets as Dataset
import torch.autograd as autograd
from torch.utils.data import Dataset, DataLoader
import torchvision
import transformers
from transformers import AlbertModel , AlbertTokenizer
import pandas  as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os
import sys
import pandas_profiling as pp
import time
import warnings
import logging
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import classification_report , accuracy_score , confusion_matrix
warnings.filterwarnings("ignore")
logging.getLogger("transformers.tokenization_utils").setLevel(logging.ERROR)
filepath = os.getcwd()+'/'
filename = 'consumer_complaints.zip'
print(' reading from path : {}'.format(filepath + filename))
start_time = time.time()
data = pd.read_csv(filepath + filename , compression = 'zip' , low_memory=False)
print('time taken to load data : {}'.format(time.time() - start_time))
data.head()
data = data[['consumer_complaint_narrative','product']]
data['seq_length'] = data['consumer_complaint_narrative'].apply(lambda x: len(str(x).split(" ")))
data.head()
data[(~data.consumer_complaint_narrative.isna())].head()
data.seq_length.describe(percentiles = [0.25 , 0.5 , 0.75 , 0.9 , 0.95 , 0.98 , 0.99])
#since 90% of data is below 64, keepong seq length at 64
seq_length = 256
data = data[(~data.consumer_complaint_narrative.isna())]
data.reset_index(inplace= True , drop=True)
data.head()
data.shape
embeddings_matrix =  np.zeros((data.shape[0] , 768))
embeddings_matrix.shape
if torch.cuda.is_available():
  device = torch.device('cuda')
  print('CUDA available using GPU')
else :
   device = torch.device('cpu')
   print('CUDA NOT available using CPU')
tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2' , max_len = seq_length)
model = AlbertModel.from_pretrained('albert-base-v2' , output_hidden_states = True)
model = model.to(device)
model = model.eval()
warnings.filterwarnings("ignore")
print_every =  500
start_time_loop = time.time()
time_list = []
for i in data.index:
  text = data.loc[i].consumer_complaint_narrative
  input_ids = torch.tensor(tokenizer.encode(text, add_special_tokens=True)[:seq_length]).unsqueeze(0)
  outputs = model(input_ids.to(device))
  last_layer = outputs[2][12][0][0]
  layer_11 = outputs[2][11][0][0]
  layer_10 = outputs[2][10][0][0]
  layer_9 = outputs[2][9][0][0]

  final_embedding = last_layer + layer_11 + layer_10 + layer_9

  embeddings_matrix[i]+=final_embedding.cpu().detach().numpy()

  if i % print_every == 0 :
    print('index {} / {} , {}'.format(i , data.shape[0] , time.time() - start_time))
    time_list.append(time.time() - start_time)
    start_time = time.time()

try:
  np.save('embeddings_matrix' , arr = embeddings_matrix , allow_pickle=True)
  print('Saved matrix locally')
except:
  print('ERROR saving embeddings')
print('time taken to generate embeddings : {}'.format(time.time() - start_time_loop))
label_encoder = LabelEncoder()
data['encoder_product'] = label_encoder.fit_transform(data['product'])
X = embeddings_matrix
Y = data.encoder_product
x_train , x_test , y_train , y_test = train_test_split(X,Y,test_size = 0.1 , random_state = 7)
print('x_train :{}'.format(x_train.shape))
print('y_train :{}'.format(y_train.shape))
print('x_test :{}'.format(x_test.shape))
print('y_test :{}'.format(y_test.shape))
batch_size = 512
#Convert input data to tensors
x_train_data_tensor = torch.tensor(x_train)
y_train_data_tensor = torch.tensor(y_train.values)
print(x_train_data_tensor.shape)

#Convert tensors to type Dataset
train_dataset = torch.utils.data.TensorDataset(x_train_data_tensor ,y_train_data_tensor)
print(type(train_dataset))

#Convert datsets to Dataloader for loading batches
train_dataset = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0)
print(type(train_dataset))

torch.save(train_dataset, 'train_dataset.dataloader')
#Convert input data to tensors
x_test_data_tensor = torch.tensor(x_test)
y_test_data_tensor = torch.tensor(y_test.values)
print(x_test_data_tensor.shape)

#Convert tensors to type Dataset
test_dataset = torch.utils.data.TensorDataset(x_test_data_tensor ,y_test_data_tensor)
print(type(train_dataset))

#Convert datsets to Dataloader for loading batches
test_dataset = DataLoader(test_dataset, batch_size=batch_size, shuffle=True, num_workers=0)
type(test_dataset)

torch.save(test_dataset, 'test_dataset.dataloader')
output_size = len(label_encoder.classes_)
batch_size = batch_size
drop_out_probability = 0.33
class ALBERT_Classification(nn.Module):
    def __init__(self , input_size , hidden_size , output_size):
        super(ALBERT_Classification ,self).__init__()
        
        self.fc_input_size = input_size
        self.fc_hidden_size = hidden_size
        self.fc_output_size = output_size
        
        
        # Inputs to hidden layer linear transformation
        self.fc1 = nn.Linear(in_features = input_size , out_features = hidden_size)
        self.fc2 = nn.Linear(in_features = input_size , out_features = hidden_size)
        self.dropout = nn.Dropout(p = 0.33)
        
        # Output layer, 10 units - one for each digit
        self.out = nn.Linear(in_features = hidden_size , out_features = output_size)
        
        
    def forward(self,x):
        
        #Pass output of BERT through Fully connected layer
        x = self.fc1(x)
        x = torch.tanh(x)

        x = self.fc2(x)
        x = torch.tanh(x)

        x = self.dropout(x)
        
        #Output layer
        x = self.out(x)
        
        return x
    
albert_classification = ALBERT_Classification(input_size = 768 
                            , hidden_size = 768 
                            , output_size = output_size )

albert_classification = albert_classification.double()
epochs = 10
loss_fn = F.cross_entropy

learning_rate = 0.001

optimizer = torch.optim.Adam(albert_classification.parameters(), lr = learning_rate)
print_every = 100
albert_classification.to(device)


# to track the average training loss per epoch as the model trains
avg_train_losses = []
# to track the average validation loss per epoch as the model trains
avg_valid_losses = [] 
patience_ctr = 0
patience = 5
    
for epoch in range(0,epochs):
    
    # to track the training loss as the model trains
    train_losses = []
    # to track the validation loss as the model trains
    valid_losses = []

    batch = 0
    #loss = 0
    start_time = time.time()
    albert_classification.train()
    for i in range(0,len(train_dataset)) :
        x, y = next(iter(train_dataset))
        x = x.to(device)
        y = y.to(device)
        
        optimizer.zero_grad()
        y_pred = albert_classification(x)
        
        #y_pred = y_pred.unsqueeze(0)
        loss = loss_fn(y_pred , y)

        loss.backward()
        optimizer.step()
        
        train_losses.append(loss.item())
        
        if (batch % print_every == 0):
            print('epoch : {} \t batch number : {} \t train loss : {}'.format(epoch, batch,loss.item()))
            
        batch +=1

        #Calculate Validation loss
    val_acc = 0
    albert_classification = albert_classification.eval()
    for j in test_dataset :
        x_v, y_v = next(iter(test_dataset))
        x_v = x_v.to(device)
        y_v = y_v.to(device)
        
        y_pred = albert_classification(x_v)
        loss = loss_fn(y_pred , y_v)
        
        valid_losses.append(loss.item())
    
    # calculate average loss over an epoch
    train_loss = np.average(train_losses)
    valid_loss = np.average(valid_losses)
    avg_train_losses.append(train_loss)
    avg_valid_losses.append(valid_loss)

    #Early stopping
    if valid_loss > train_loss:
        patience_ctr +=1
        print('patience_ctr : {}'.format(patience_ctr))
        if patience_ctr >= patience:
            print('early stoppping since valid_loss > train_loss')
            break
    else :
        patience_ctr = 0
    
    epoch_len = len(str(epochs))
    print('epoch : {}\t train_loss :{}\t validation loss : {}\n'.format(epoch , train_loss , valid_loss))
    
            
    print('epoch : {} \t train loss : {} \t time required : {}'.format(epoch, loss.item(),(time.time() - start_time)))
st_time = time.time()
ctr = 0
print_every = 5
albert_classification = albert_classification.double()
albert_classification = albert_classification.to(device)
albert_classification = albert_classification.eval()
for j in test_dataset :
    x_v, y_v = next(iter(test_dataset))
    x_v = x_v.to(device)
    #y_v = y_v.to(device)
   
    op = albert_classification(x_v)
    op = torch.argmax(F.softmax(op), dim = 1)
    if ctr == 0:
        y_pred = op
        y_actual = y_v
    else:
        y_pred = torch.cat((y_pred , op))
        y_actual = torch.cat((y_actual , y_v))
   
    ctr += 1
   
    if ctr % print_every == 0:
        print(ctr)
print('time taken for prediction :{} seconds'.format(time.time() - st_time ))
def my_classification_report(y_actual , y_predicted):
   
    print(classification_report(y_actual, y_pred, target_names=list(label_encoder.classes_)))
    acc = accuracy_score(y_actual , y_predicted)
    print('accuracy : {}'.format(acc))
   
    conf_mat=confusion_matrix(y_actual, y_pred)
    print(conf_mat)
    #plt.figure(figsize=(20,20))
    ax = plt.subplot()
    sns.heatmap(conf_mat, annot=True, ax = ax); #annot=True to annotate cells

    # labels, title and ticks
    ax.set_xlabel('Predicted labels')
    ax.set_ylabel('True labels')
    ax.set_title('Confusion Matrix')
    ax.xaxis.set_ticklabels(list(label_encoder.classes_))
    ax.yaxis.set_ticklabels(list(label_encoder.classes_))
    plt.xticks(rotation=90)
    plt.yticks(rotation=0)
    #plt.savefig('conf_matrix.png')
    plt.show()

#y_actual = y_test.cpu().detach().numpy()
#y_pred = y_pred.cpu().detach().numpy()
my_classification_report(y_actual = y_actual , y_predicted = y_pred)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11543398834798065, 'fraction_numerical': 0.012738224548237385, 'mean_word_length': 3.748241912798875, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '475268', 'license': 'MIT', 'n_tokens_mistral': 3497, 'n_tokens_neox': 3366, 'n_words': 713, 'path': 'text_classification_using_transformers.ipynb', 'repo_name': 'Adi8885/portfolio', 'size': '10127'}