<jupyter_start><jupyter_text><jupyter_code># This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# Any results you write to the current directory are saved as output.
import pandas as pd
import numpy as np
from torch.utils.data import Dataset, DataLoader
from sklearn.model_selection import train_test_split
tweet = pd.read_csv('train.csv')
tweet.head()<jupyter_output><empty_output><jupyter_text>New Section  Data ExplorationWe see 3 columns let us find the information on the data<jupyter_code>import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
plt.figure(figsize = (12,8))
sns.countplot(tweet['target'])
plt.title("Real or not count", size = 16)
plt.xlabel(xlabel='Two target categories', size = 12)
plt.ylabel(ylabel='Number of reviews', size = 12)<jupyter_output><empty_output><jupyter_text>Let us find out if the text length is of any significance in identifying the fake or real<jupyter_code>tweet['target'].value_counts()
plt.figure(figsize = (12,8))
sns.distplot(tweet[tweet['target']==0]['text'].apply(lambda x: len(x)), bins = 100, label = "Target 0")
sns.distplot(tweet[tweet['target']==1]['text'].apply(lambda x: len(x)), bins = 100, label = "Target 1")
plt.title("Distribution of length of two label from length between {} to {} words".format(min(tweet['text'].apply(lambda x: len(x))),
                                                                            max(tweet['text'].apply(lambda x: len(x)))), size = 20)
plt.legend()
plt.axvline(x = np.mean(tweet[tweet['target']==0]['text'].apply(lambda x: len(x))), color = 'blue')
plt.axvline(x = np.mean(tweet[tweet['target']==1]['text'].apply(lambda x: len(x))), color = 'red')

plt.xlabel(xlabel='Word count in each tweet', size = 12)

import numpy as np 

plt.figure(figsize = (10,6))
sns.distplot(tweet[tweet['target']==0]['text'].apply(lambda x: np.log(len(x))), bins = 50, label = "target 0")
sns.distplot(tweet[tweet['target']==1]['text'].apply(lambda x: np.log(len(x))), bins = 50, label = "target 1")
plt.title("Log distribution of length of two label", size = 20)
plt.legend()
plt.axvline(x = np.mean(tweet[tweet['target']==0]['text'].apply(lambda x: np.log(len(x)))), color = 'blue')
plt.axvline(x = np.mean(tweet[tweet['target']==1]['text'].apply(lambda x: np.log(len(x)))), color = 'red')<jupyter_output><empty_output><jupyter_text>Let us apply t-test to see whether the difference is of any statistical significance<jupyter_code>import numpy as np
from scipy.stats import ttest_ind, ttest_ind_from_stats
from scipy.special import stdtr
a = tweet[tweet['target']==0]['text'].apply(lambda x: np.log(len(x))).values
b = tweet[tweet['target']==1]['text'].apply(lambda x: np.log(len(x))).values
ttest_ind(a, b, axis=0, equal_var=False, nan_policy='omit')<jupyter_output><empty_output><jupyter_text>The t-test proves that it could be good idea to put the length of the tweet as a feature for prediction let us try to plot the confidence interval<jupyter_code># t-test for independent variables with unequal variance
import statsmodels.stats.weightstats as ws
means = ws.CompareMeans(ws.DescrStatsW(a), ws.DescrStatsW(b))
confint = means.tconfint_diff(alpha=0.01, alternative='two-sided', usevar='unequal') # confidence with 95%
degfree = means.dof_satt()

plt.figure(figsize = (12,8))
plt.hist(tweet[tweet['target']==0]['text'].apply(lambda x: len(x)), bins = 100, label = "Target 0", alpha = 0.5)
plt.hist(tweet[tweet['target']==1]['text'].apply(lambda x: len(x)), bins = 100, label = "Target 1", alpha = 0.5)
plt.title("Tweet length between {} to {} words".format(min(tweet['text'].apply(lambda x: len(x))),
                                                                            max(tweet['text'].apply(lambda x: len(x)))), size = 20)
plt.legend()
plt.axvline(x = np.mean(tweet[tweet['target']==0]['text'].apply(lambda x: len(x))), color = 'blue', alpha = 0.5)
plt.axvline(x = np.mean(tweet[tweet['target']==1]['text'].apply(lambda x: len(x))), color = 'red', alpha = 0.5)
# adding confidence interval
plt.axvline(x = (np.mean(tweet[tweet['target']==1]['text'].apply(lambda x: len(x)))-confint[1]), color = 'red',
            linestyle='--',alpha = 0.5)
plt.axvline(x = (np.mean(tweet[tweet['target']==1]['text'].apply(lambda x: len(x)))+confint[0]), color = 'red',
            linestyle='--',alpha = 0.5)
plt.xlabel(xlabel='Word count in each tweet', size = 12)<jupyter_output><empty_output><jupyter_text>Stats tell us that it would be a good idea to keep length as feature. Let us also try to visualize this with boxplot<jupyter_code>plt.figure(figsize = (10,6))
sns.boxplot(x = 'target', y = tweet['text'].apply(lambda x: len(x)), data = tweet)
plt.title("Boxplot visualization for 2 target classes", size = 16)<jupyter_output><empty_output><jupyter_text>Seeing the distribution and the statistics let us add the tweet length as one of the features  Data Preprocessing<jupyter_code>def twt_length(text_col):
    return text_col.apply(lambda x: len(x))    
tweet['length'] = twt_length(tweet['text'])
tweet.head()
tweet['id'].nunique()
tweet.shape
tweet['keyword'].value_counts()
tweet.isnull().sum()<jupyter_output><empty_output><jupyter_text>As there are no missing values we'll make process the text<jupyter_code>from nltk.tokenize import word_tokenize
import nltk 
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('stopwords')
nltk.download('averaged_perceptron_tagger')
# Step - a : Change all the text to lower case. This is required as python interprets 'dog' and 'DOG' differently
tweet['text'] = [str.lower(txt) for txt in tweet['text']]

# Step - b : Tokenization : In this each entry in the corpus will be broken into set of words
tweet['text'] = [word_tokenize(txt) for txt in tweet['text']]

tweet.head(3)
from nltk.corpus import stopwords
# Step - c : Remove Stop words, Non-Numeric and perfom Word Stemming/Lemmenting.
tweet['stp_text'] = tweet['text'].apply(lambda x: [word for word in x if (word not in stopwords.words('english'))& (word.isalpha())])
import nltk
from nltk import pos_tag
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet as wn
#Step - e
# WordNetLemmatizer requires Pos tags to understand if the word is noun or verb or adjective etc. By default it is set to Noun
"""
1. pos tag function
2. Define a function for lemmatizing
"""
def get_wordnet_pos(word):
    """Map POS tag to first character lemmatize() accepts"""
    tag = nltk.pos_tag([word])[0][1][0].upper()
    tag_dict = {"J": wn.ADJ,
                "N": wn.NOUN,
                "V": wn.VERB,
                "R": wn.ADV}

    return tag_dict.get(tag, wn.NOUN)

def word_lemmatizer(series):
  """lemmatizes words in series to their root using POS tag"""
  
  count = series.shape[0] # number of inputs
  txt_series = series.copy()
  lemmatizer = WordNetLemmatizer() # initializing lemmatizer
  
  for i in range(count):
    tst = [lemmatizer.lemmatize(w, get_wordnet_pos(w)) for w in txt_series.iloc[i]]
    txt_series.iloc[i] = tst
  
  return(txt_series)

# a new column with lemmatized words
tweet['lem_txt'] = word_lemmatizer(tweet['stp_text'])
tweet.head()
#step f
#The above dataframe gives us the list of final text but for TF-IDF we need the input as string. So we can create a final column with lambda function
tweet['text_final'] = tweet['lem_txt'].apply(lambda x: ' '.join(x))
tweet['text_final']
from sklearn import model_selection, svm
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report<jupyter_output><empty_output><jupyter_text>Prepare Evaluate/ Train Datasets<jupyter_code>train_X, val_X, train_y, val_y = model_selection.train_test_split(tweet['text_final'],tweet['target'],test_size=1/3, random_state = 10)
Tfidf_vect = TfidfVectorizer(max_features=10000)
Tfidf_vect.fit(train_X)
Train_X_Tfidf = Tfidf_vect.transform(train_X)
Val_X_Tfidf = Tfidf_vect.transform(val_X)
train_X.head()
Tfidf_vect.get_feature_names()[5000:5010]
# Way to create image from simple word count
from wordcloud import WordCloud, STOPWORDS
words = ' '.join(tweet[tweet['target'] ==0]['text_final'])
wordcloud = WordCloud(stopwords=STOPWORDS,
                      background_color='black',
                      width=3000,
                      height=2500
                     ).generate(words)
                     
plt.figure(figsize=(12, 12))
plt.imshow(wordcloud)
plt.title("Word cloud image for fake tweets", size = 32)
plt.axis('off')
plt.show()
words = ' '.join(tweet[tweet['target'] ==1]['text_final'])
wordcloud = WordCloud(stopwords=STOPWORDS,
                      background_color='white',
                      width=3000,
                      height=2500
                     ).generate(words)
                     
plt.figure(figsize=(12, 12))
plt.imshow(wordcloud)
plt.title("Word cloud image for real tweets", size = 32)
plt.axis('off')
plt.show()<jupyter_output><empty_output><jupyter_text>ML Models<jupyter_code>from sklearn.linear_model import LogisticRegression
LR = LogisticRegression()
LR.fit(Train_X_Tfidf,train_y)
predictions_LR = LR.predict(Val_X_Tfidf)
print("LR Accuracy Score -> ",accuracy_score(predictions_LR, val_y)*100)
import sklearn.metrics as metrics
# calculate the fpr and tpr for all thresholds of the classification
probs = LR.predict_proba(Val_X_Tfidf)
preds = probs[:,1]
fpr, tpr, threshold = metrics.roc_curve(val_y, preds)
roc_auc = metrics.auc(fpr, tpr)

# method I: plt
import matplotlib.pyplot as plt
plt.figure(figsize = (10,8))
plt.title('Receiver Operating Characteristic for Logistic Regression', size = 16)
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()
from sklearn.metrics import precision_recall_curve

# predict probabilities
lr_probs = LR.predict_proba(Val_X_Tfidf)
# keep probabilities for the positive outcome only
lr_probs = lr_probs[:, 1]
# predict class values
yhat = LR.predict(Val_X_Tfidf)
lr_precision, lr_recall, _ = precision_recall_curve(val_y, lr_probs)
lr_f1, lr_auc = metrics.f1_score(val_y, yhat), metrics.auc(lr_recall, lr_precision)
# summarize scores
print('Logistic: f1=%.3f auc=%.3f' % (lr_f1, lr_auc))
# plot the precision-recall curves
no_skill = len(val_y[val_y==1]) / len(val_y)
plt.figure(figsize = (10,8))
plt.plot([0, 1], [no_skill, no_skill], linestyle='--', label='No Skill')
plt.plot(lr_recall, lr_precision, label='Logistic')
# axis labels
plt.title('Precision Recal Curve', size = 16)
plt.xlabel('Recall')
plt.ylabel('Precision')
# show the legend
plt.legend()<jupyter_output>Logistic: f1=0.748 auc=0.845<jupyter_text>Lets try with SVM<jupyter_code># Classifier - Algorithm - SVM
# fit the training dataset on the classifier
SVM = svm.SVC(C=1.0, kernel='linear')
SVM.fit(Train_X_Tfidf,train_y)
# predict the labels on validation dataset
predictions_SVM = SVM.predict(Val_X_Tfidf)
# Use accuracy_score function to get the accuracy
print("SVM Accuracy Score -> ",accuracy_score(predictions_SVM, val_y)*100)
confusion_matrix(val_y,predictions_SVM)<jupyter_output><empty_output><jupyter_text>Let's try to optimize the loss function using hyperopt<jupyter_code>from hyperopt import hp, fmin, tpe, Trials, STATUS_OK
# defining the space for hyperparameter tuning
space={'C': hp.uniform('C', 0.1, 100),
        'gamma': hp.uniform('gamma', 0.1, 15),
        'degree' : hp.quniform('degree', 1,10,1),
        'kernel': 'linear'
    }
def hyperparameter_tuning(space):
    SVM=svm.SVC(C =space['C'], 
                gamma = space['gamma'],
                degree = space['degree'],
                kernel = space['kernel'])
        
    SVM.fit(Train_X_Tfidf, train_y)

    pred = SVM.predict(Val_X_Tfidf)
    accuracy = accuracy_score(val_y, pred>0.5)
    print ("SCORE:", accuracy)
    #change the metric if you like
    return {'loss': -accuracy, 'status': STATUS_OK }
# run the hyper paramter tuning
trials = Trials()
best = fmin(fn=hyperparameter_tuning,
            space=space,
            algo=tpe.suggest,
            max_evals=50,
            trials=trials)

print (best)
SVM=svm.SVC(C =best['C'], 
            gamma = best['gamma'],
            degree = best['degree'],
            kernel = 'linear')
SVM.fit(Train_X_Tfidf,train_y)
# predict the labels on validation dataset
predictions_SVM = SVM.predict(Val_X_Tfidf)

# Use accuracy_score function to get the accuracy
print("SVM Accuracy Score -> ",accuracy_score(predictions_SVM, val_y)*100)
import numpy
def plot_coefficients(classifier, feature_names, top_features=20):
 coef = classifier.coef_.toarray().reshape(-1)
 top_positive_coefficients = np.argsort(coef)[-top_features:]
 top_negative_coefficients = np.argsort(coef)[:top_features]
 top_coefficients = np.hstack([top_negative_coefficients, top_positive_coefficients])
 # create plot
 plt.figure(figsize=(15, 5))
 colors = ['red' if c < 0 else 'blue' for c in coef[top_coefficients]]
 plt.bar(np.arange(2 * top_features), coef[top_coefficients], color=colors)
 feature_names = np.array(feature_names)
 plt.xticks(np.arange(1, 1 + 2 * top_features), feature_names[top_coefficients], rotation=60, ha='right')
 plt.show()

plot_coefficients(SVM, Tfidf_vect.get_feature_names())<jupyter_output><empty_output><jupyter_text>NLP with Catboost The catboost model has novel approach to also take othe columns into consideration<jupyter_code>!pip install catboost
from catboost import Pool, CatBoostClassifier
tweet.head()
cat_features = ['keyword']
text_features = ['text_final']
# we can create a new test model
train_X, val_X, train_y, val_y = model_selection.train_test_split(tweet[['text_final','keyword','length']],tweet['target'],test_size=1/3, random_state = 10)
#fill NaN in keyword with blank space
def fill_na(df, features):
  for feature in features:
    df[feature].fillna('', inplace=True)

fill_na(train_X,cat_features)
fill_na(val_X,cat_features)
train_X.isnull().sum()

def real_or_not_tweet(X_train, X_test, y_train, y_test, catboost_params={}, verbose=200):
    learn_pool = Pool(
        X_train, 
        y_train, 
        cat_features=cat_features,
        text_features=text_features,
        feature_names=list(X_train)
    )
    test_pool = Pool(
        X_test, 
        y_test, 
        cat_features=cat_features,
        text_features=text_features,
        feature_names=list(X_train)
    )
    
    catboost_default_params = {
        'iterations': 2000,
        'learning_rate': 0.03,
        'eval_metric': 'Accuracy',
        'task_type': 'GPU'
        
    }
    
    catboost_default_params.update(catboost_params)
    
    model = CatBoostClassifier(**catboost_default_params)
    model.fit(learn_pool, eval_set=test_pool, verbose=verbose, 
              early_stopping_rounds = 100)

    return model
cbc = real_or_not_tweet(train_X, val_X, train_y, val_y)<jupyter_output>0:	learn: 0.7718227	test: 0.7947203	best: 0.7947203 (0)	total: 43.6ms	remaining: 1m 27s
bestTest = 0.8073286052
bestIteration = 88
Shrink model to first 89 iterations.<jupyter_text>We see Catboost is performing slightly better than the svm. Let us try to tune the model<jupyter_code>from sklearn.metrics import accuracy_score, f1_score
learn_pool = Pool(train_X, train_y, cat_features=cat_features, 
                  text_features=text_features, feature_names=list(train_X))
val_pool = Pool( val_X, val_y, cat_features=cat_features,
                 text_features=text_features, feature_names=list(train_X))

def objective(space):
    # Instantiate the classifier
    clf = CatBoostClassifier(learning_rate = space['learning_rate'], 
                                 n_estimators =int(space['n_estimators']), 
                                 max_depth = int(space['max_depth']),
                                 reg_lambda = int(space['reg_lambda']),
                                 
                                 bagging_temperature = space['bagging_temperature'],
                             task_type = 'GPU')
    
    
    # Fit the classsifier
    clf.fit(learn_pool,
            eval_set=val_pool, 
            early_stopping_rounds=10,verbose=False)
    
    # Predict on Cross Validation data
    pred = clf.predict(val_X)
    
    # Calculate our Metric - accuracy
    fscore = f1_score(val_y, pred>0.5)
# return needs to be in this below format. We use negative of accuracy since we want to maximize it.
    return {'loss': -fscore, 'status': STATUS_OK }

space =    {
     'learning_rate':     hp.choice('learning_rate',     np.arange(0.01, 0.8, 0.05)),
     'max_depth':         hp.choice('max_depth',         np.arange(2, 16, 1, dtype=int)),
     'colsample_bylevel': hp.uniform('colsample_bylevel', 0.5, 1.0),
     'n_estimators':      hp.quniform('n_estimators', 500, 5000,100),
     'reg_lambda' :       hp.uniform('reg_lambda', 0,5),
     'bagging_temperature': hp.uniform('bagging_temperature', 0.0, 100)

    }


trials = Trials()
best = fmin(fn=objective,
            space=space,
            algo=tpe.suggest,
            max_evals=40,
            trials=trials)
print(best)<jupyter_output>100%|██████████| 40/40 [11:34<00:00, 14.18s/it, best loss: -0.7616279069767441]
{'bagging_temperature': 13.528001661404787, 'colsample_bylevel': 0.5619511092230439, 'learning_rate': 6, 'max_depth': 6, 'n_estimators': 2100.0, 'reg_lambda': 0.5587961367147581}<jupyter_text>In the above optimization we tried to find hyper parameters for best f1 score for which we have got the value of 0.76 We see that the base model with less data are performing best with upto 80% of accuracy. To get better results we'll try the pretrained model like BERT<jupyter_code>!pip install transformers
from transformers import BertModel, BertTokenizer, BertForSequenceClassification
bert_model = BertModel.from_pretrained('bert-base-uncased')
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
from torch.utils.data import Dataset, DataLoader

class LoadDataset(Dataset):

    def __init__(self, filename, maxlen):

        # Store the contents of the file in a pandas dataframe
        self.df = pd.read_csv(filename, delimiter=',')

        # Initialize the BERT tokenizer
        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

        # Define the Maxlength for padding/truncating
        self.maxlen = maxlen

    def __len__(self):
        return len(self.df)

    def __getitem__(self, index):

        # Selecting the sentence and label at the specified index in the data frame
        sentence = self.df.loc[index, 'text']
        label = self.df.loc[index, 'target']

        # Tokenize the sentence
        tokens = self.tokenizer.tokenize(sentence)

        # Inserting the CLS and SEP token at the beginning and end of the sentence
        tokens = ['[CLS]'] + tokens + ['[SEP]']
        
        # Padding/truncating the sentences to the maximum length
        if len(tokens) < self.maxlen:
            tokens = tokens + ['[PAD]' for _ in range(self.maxlen - len(tokens))]
        else:
            tokens = tokens[:self.maxlen-1] + ['[SEP]']
        
        # Convert the sequence to ids with BERT Vocabulary
        tokens_ids = self.tokenizer.convert_tokens_to_ids(tokens)
        
        # Converting the list to a pytorch tensor
        tokens_ids_tensor = torch.tensor(tokens_ids)

        # Obtaining the attention mask
        attn_mask = (tokens_ids_tensor != 0).long()

        return tokens_ids_tensor, attn_mask, label
import pandas as pd

bert_df = pd.read_csv('train.csv')
bert_df.head()
#train, val = train_test_split(bert_df[['text','target']],test_size=1/3, random_state = 10)


train.to_csv('learn.csv')

val.to_csv('val.csv')
# Creating instances of training and validation set
train_set = LoadDataset( 'learn.csv', maxlen = 64)
val_set = LoadDataset('val.csv', maxlen = 64)
# Creating intsances of training and validation dataloaders
train_loader = DataLoader(train_set, batch_size = 16, num_workers = 5)
val_loader = DataLoader(val_set, batch_size = 16, num_workers = 5)<jupyter_output><empty_output><jupyter_text>Building the model<jupyter_code>from torch import nn
class RealOrNotClassifier(nn.Module):

    def __init__(self, freeze_bert = True):
        super(RealOrNotClassifier, self).__init__()

        # Instantiating the BERT model object 
        self.bert_layer = BertModel.from_pretrained('bert-base-uncased')
        
        # Defining layers like dropout and linear
        self.dropout = nn.Dropout(0.1)
        self.classifier = nn.Linear(768, 1)

    def forward(self, seq, attn_masks):
        '''
        Inputs:
            -seq : Tensor of shape [B, T] containing token ids of sequences
            -attn_masks : Tensor of shape [B, T] containing attention masks to be used to avoid contibution of PAD tokens
        '''

        # Getting contextualized representations from BERT Layer
        cont_reps, _ = self.bert_layer(seq, attention_mask = attn_masks)

        # Obtaining the representation of [CLS] head
        cls_rep = cont_reps[:, 0]
        # print('CLS shape: ',cls_rep.shape)

        # Feeding cls_rep to the classifier layer
        logits = self.classifier(cls_rep)
        # print('Logits shape: ',logits.shape)

        return logits
model = RealOrNotClassifier()
import torch
from torch.optim import Adam
from torch.nn import BCEWithLogitsLoss

criterion = BCEWithLogitsLoss()
optimizer = Adam(model.parameters(), lr = 2e-5)
if torch.cuda.is_available():
  device = 'cuda'
else:
  device = 'cpu'

print(device)

# Defining a function for calculating accuracy
def logits_accuracy(logits, labels):
    probs = torch.sigmoid(logits.unsqueeze(-1))
    preds = (probs > 0.5).long()
    acc = (preds.squeeze() == labels).float().mean()
    return acc
# Defining an evaluation function for training 
def evaluate(net, criterion, val_loader, device):
  
    losses, accuracies = 0, 0
    
    # Setting model to evaluation mode
    net.eval()

    count = 0
    for (seq, attn_masks, labels) in val_loader:
        count += 1

        # Move inputs and targets to device
        seq, attn_masks, labels = seq.to(device), attn_masks.to(device), labels.to(device)

        # Get logit predictions
        val_logits = net(seq, attn_masks)

        # Calculate loss
        val_loss = criterion(val_logits.squeeze(-1), labels.float())
        losses += val_loss.item()

        # Calculate validation accuracy
        accuracies += logits_accuracy(val_logits, labels)

    return losses / count, accuracies / count
from time import time
def train(net, criterion, optimizer, train_loader, val_loader, device, epochs=4, print_every=100):
    
    # Move model to device
    net.to(device)
    # Setting model to training mode
    net.train()

    print('========== ========== STARTING TRAINING ========== ==========')

    for epoch in range(epochs):

        print('\n\n========== EPOCH {} =========='.format(epoch))        
        t1 = time()

        for i, (seq, attn_masks, labels) in enumerate(train_loader):

            # Clear gradients
            optimizer.zero_grad()  

            # Moving tensors to device
            seq, attn_masks, labels = seq.to(device), attn_masks.to(device), labels.to(device)

            # Obtaining the logits from the model
            logits = net(seq,attn_masks)

            # Calculating the loss
            loss = criterion(logits.squeeze(-1), labels.float())

            # Backpropagating the gradients
            loss.backward()

            # Clipping gradients to tackle exploding gradients
            nn.utils.clip_grad_norm_(net.parameters(), 1)

            # Optimization step
            optimizer.step()

            if (i + 1) % print_every == 0:
                print("Iteration {} ==== Loss: {}".format(i+1, loss.item()))

        t2 = time()
        print('Time Taken for Epoch: {}'.format(t2-t1))
        print('\n========== Validating ==========')
        mean_val_loss, mean_val_acc = evaluate(net, criterion, val_loader, device)
        print("Validation Loss: {}\nValidation Accuracy: {}".format(mean_val_loss, mean_val_acc))
%%time
# starting training
train(model, criterion, optimizer, train_loader, val_loader, device, epochs=4, print_every=100)
# Saving our model
import os
save_path = '/content/drive/My Drive/Colab Notebooks/bert'
if not os.path.isdir(save_path):
    os.mkdir(save_path)

torch.save({
    'model_state_dict': model.state_dict(),
    'optimizer_state_dict': optimizer.state_dict()
}, '/content/drive/My Drive/Colab Notebooks/bert/rnmodel1.pth')
# Loading the checkpoints for resuming training
checkpoint = torch.load('/content/drive/My Drive/Colab Notebooks/bert/rnmodel1.pth')
model.load_state_dict(checkpoint['model_state_dict'])

import torch
#predictions
# predictor
inference_file = torch.load('/content/drive/My Drive/Colab Notebooks/bert/rnmodel.pth')
predictor = RealOrNotClassifier()
predictor.load_state_dict(inference_file['model_state_dict'])
def preprocess(sentence, maxlen=64):

    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

    # Tokenize the sentence
    tokens = tokenizer.tokenize(sentence)

    # Inserting the CLS and SEP token at the beginning and end of the sentence
    tokens = ['[CLS]'] + tokens + ['[SEP]']
    
    # Padding/truncating the sentences to the maximum length
    if len(tokens) < maxlen:
        tokens = tokens + ['[PAD]' for _ in range(maxlen - len(tokens))]
    else:
        tokens = tokens[:maxlen-1] + ['[SEP]']
    
    # Convert the sequence to ids with BERT Vocabulary
    tokens_ids = tokenizer.convert_tokens_to_ids(tokens)
    
    # Converting the list to a pytorch tensor
    tokens_ids_tensor = torch.tensor(tokens_ids).unsqueeze(0)

    # Obtaining the attention mask
    attn_mask = (tokens_ids_tensor != 0).long()

    return tokens_ids_tensor, attn_mask
# Defining an evaluation function for training 
def predict(net, iseq, masks):
    device = 'cpu'
    # Setting model to evaluation mode
    net.eval()

    # Move inputs and targets to device
    iseq, masks = iseq.to(device), masks.to(device)

    # Get logit predictions
    p_logit = net(iseq, masks)

    probs = torch.sigmoid(p_logit.unsqueeze(-1))
    preds = (probs > 0.5).long().squeeze(0)

   
    return preds, probs
test_tokens, test_attn = comb(" Metro College of technology has helped me increase my knowledge for design and manufacture and be competitive in the market.")
float(test_attn.detach().numpy())
def comb(text):
  test_tokens, test_attn = preprocess(text)
  preds, probs = predict(predictor,test_tokens, test_attn)
  return (int(preds.numpy()), probs.detach().numpy())

trgt = []
prb = []
for twt in val.text.values:
  pred, proba = comb(twt)
  trgt.append(pred)
  prb.append(proba)
val = pd.read_csv('val.csv')
l = val.shape[0]
#l = 163
n = 0
p_list = pd.Series([])
while l/50 > 1:
  
  p_list = p_list.append(val['text'][n:n+50].apply(lambda x: comb(x)))
  n = n+50
  l = l-50
else:
  p_list = p_list.append(val['text'][0:l].apply(lambda x: comb(x)))

p_list<jupyter_output><empty_output><jupyter_text><jupyter_code>bert_df.head()
import re
text1 = 'RT @UniversalIND: #F8 is now playing in the theaters near you! So hurry and book your tickets https://www.abcabcabc.com'
text1 = re.sub(r'@\S+|http?s?://\S+', '', text1)

print(text1)
def clean_tweet(text):
  text = re.sub(r'@\S+|http?s?://\S+', '', text)
  return(text)

bert_df['cln_twt'] = bert_df['text'].apply(lambda x: clean_tweet(x))
pd.set_option('display.max_colwidth', -1)
bert_df.tail(5)
# Step - a : Change all the text to lower case. This is required as python interprets 'dog' and 'DOG' differently
bert_df['cln_twt1'] = [str.lower(txt) for txt in bert_df['cln_twt']]

# Step - b : Tokenization : In this each entry in the corpus will be broken into set of words
bert_df['cln_twt1'] = [word_tokenize(txt) for txt in bert_df['cln_twt1']]

bert_df.head(3)
bert_df.to_csv('clean_txt.csv')
bert_df[bert_df.duplicated('text')]['target'].value_counts()
# Step - c : Remove Stop words, Non-Numeric and perfom Word Stemming/Lemmenting.
bert_df['stp_text'] = bert_df['cln_twt1'].apply(lambda x: [word for word in x if (word not in stopwords.words('english'))& (word.isalpha())])
bert_df.head()
#Step - e
# WordNetLemmatizer requires Pos tags to understand if the word is noun or verb or adjective etc. By default it is set to Noun
"""
1. pos tag function
2. Define a function for lemmatizing
"""
def get_wordnet_pos(word):
    """Map POS tag to first character lemmatize() accepts"""
    tag = nltk.pos_tag([word])[0][1][0].upper()
    tag_dict = {"J": wn.ADJ,
                "N": wn.NOUN,
                "V": wn.VERB,
                "R": wn.ADV}

    return tag_dict.get(tag, wn.NOUN)

def word_lemmatizer(series):
  """lemmatizes words in series to their root using POS tag"""
  
  count = series.shape[0] # number of inputs
  txt_series = series.copy()
  lemmatizer = WordNetLemmatizer() # initializing lemmatizer
  
  for i in range(count):
    tst = [lemmatizer.lemmatize(w, get_wordnet_pos(w)) for w in txt_series.iloc[i]]
    txt_series.iloc[i] = tst
  
  return(txt_series)

# a new column with lemmatized words
bert_df['lem_txt'] = word_lemmatizer(bert_df['stp_text'])
bert_df.head()
#step f
#The above dataframe gives us the list of final text but for TF-IDF we need the input as string. So we can create a final column with lambda function
bert_df['text_final'] = bert_df['cln_twt1'].apply(lambda x: ' '.join(x))
bert_df.head()
bert_df.drop_duplicates('text',inplace = True)
val['pred'] = trgt
val['prob'] = prb
from sklearn.metrics import confusion_matrix, classification_report
print(classification_report(val['target'],val['pred']))
confusion_matrix(val['target'],val['pred'])<jupyter_output><empty_output><jupyter_text>let us see the kind of tweets model is unable to guess right<jupyter_code>pd.set_option('display.max_colwidth', -1)
val[val['target'] != val['pred']].to_csv('misclassified')

trgt = []
prb = []
for twt in val.text.values:
  pred, proba = comb(twt)
  trgt.append(pred)
  prb.append(proba)
val['pred'] = trgt
val['prob'] = prb
test = pd.read_csv('test.csv')
test['cln_twt'] = test['text'].apply(lambda x: clean_tweet(x))
# Step - a : Change all the text to lower case. This is required as python interprets 'dog' and 'DOG' differently
test['cln_twt1'] = [str.lower(txt) for txt in test['cln_twt']]

# Step - b : Tokenization : In this each entry in the corpus will be broken into set of words
test['cln_twt1'] = [word_tokenize(txt) for txt in test['cln_twt1']]

test['text_final'] = test['cln_twt1'].apply(lambda x: ' '.join(x))

trgt = []
prb = []
for twt in test.text_final.values:
  pred, proba = comb(twt)
  trgt.append(pred)
  prb.append(proba)
test['pred'] = trgt
test['prob'] = prb
test.head()
test.to_csv('results')
submission = test[['id','pred']]
submission.columns = ['id','target']
submission.to_csv('submission.csv', index = False, header = True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12962962962962962, 'fraction_numerical': 0.01690408357075024, 'mean_word_length': 3.9531201003449357, 'pattern_counts': {'":': 8, '<': 74, '<?xml version=': 0, '>': 78, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '544423', 'license': 'MIT', 'n_tokens_mistral': 11248, 'n_tokens_neox': 10579, 'n_words': 2690, 'path': 'NLP_SVM_CB_BERT.ipynb', 'repo_name': 'kshitijmamgain/NLP', 'size': '31590'}