<jupyter_start><jupyter_text>Task1  Importing the required libraries.<jupyter_code>import pandas as pd
import re
import nltk
import string
import numpy as np
import itertools
import matplotlib.pyplot as plt
import keras

from wordcloud import WordCloud
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import Embedding,Conv1D,MaxPooling1D,Bidirectional,LSTM,Dropout,Dense,ActivityRegularization
from keras.losses import BinaryCrossentropy
from keras import regularizers
from keras import optimizers
from keras.callbacks import EarlyStopping 
from sklearn.metrics import confusion_matrix,matthews_corrcoef,accuracy_score<jupyter_output><empty_output><jupyter_text>Reading the data as a pandas dataframe.<jupyter_code>raw_ds = pd.read_csv('CVAssignmentDataset.csv')
raw_ds.head()<jupyter_output><empty_output><jupyter_text>Exploring the data.<jupyter_code>pd.set_option('display.max_colwidth', None) #To get the maximum width of column.
raw_ds.head()
raw_ds.describe()<jupyter_output><empty_output><jupyter_text>Dropping Nan value.<jupyter_code>print(raw_ds.isna().sum())
raw_ds.dropna(inplace=True)
raw_ds['Class'].value_counts() # To get unique values of 'Class' feature.<jupyter_output><empty_output><jupyter_text>Assigning text samples and labels.<jupyter_code>text_samples = raw_ds['Text']
labels = raw_ds['Class']
labels = labels.apply(lambda x:int(x)) # Converting string labels into integer type.<jupyter_output><empty_output><jupyter_text>Preprocessing functions to clean, tokenize and pad the data.<jupyter_code>from nltk.corpus import stopwords
from bs4 import BeautifulSoup
Stopwords=set(stopwords.words('english'))

def clean_data(text):
    text = re.sub(r"http\S+", "", text) #Removes links in the text
    text = re.sub(r"@", "at",text) #Converts '@' symbol to 'at'
    text = re.sub(r'[^\x00-\x7f]','', text) # Removes all Non-ASCII characters.
    html = BeautifulSoup(text,'lxml') #Removes html character references like '&amp' 
    text = html.get_text()
    
    text = text.strip().lower() 
    
    punctuation = string.punctuation
    punctuation_dict = dict((c,"") for c in punctuation)
    punctuation_map = str.maketrans(punctuation_dict)
    text = text.translate(punctuation_map)  #Removes punctuation from text.
    
    text = ' '.join(word for word in text.split() if word not in Stopwords)
    
    return text
def tokenization(standardized_text,Max_vocab,Max_length):   
    tokenizer = keras.preprocessing.text.Tokenizer(num_words=Max_vocab,filters=string.punctuation)
    tokenizer.fit_on_texts(standardized_text)

    tokenized_text = tokenizer.texts_to_sequences(standardized_text)
    padded_tokenized_text = keras.preprocessing.sequence.pad_sequences(tokenized_text,maxlen=Max_length)
    
    return padded_tokenized_text,tokenizer<jupyter_output><empty_output><jupyter_text>Splitting the data into train and test sets.<jupyter_code>X_train,X_test,y_train,y_test = train_test_split(text_samples,labels,train_size=0.9,random_state=123) # There are enough samples for us to do 90:10 split.
print('Shape of X_train :',X_train.shape)
print('Shape of X_test  :',X_test.shape)
print('Shape of y_train :',y_train.shape)
print('Shape of y_test  :',y_test.shape)<jupyter_output>Shape of X_train : (77814,)
Shape of X_test  : (8646,)
Shape of y_train : (77814,)
Shape of y_test  : (8646,)<jupyter_text>Cleaning data.<jupyter_code>train_cleaned = X_train.apply(clean_data)
test_cleaned = X_test.apply(clean_data)<jupyter_output><empty_output><jupyter_text>Tokenizing and padding data.<jupyter_code>Max_vocab = 10000
Max_length = 50
Embedding_dim= 32
train_samples,tokenizer = tokenization(train_cleaned,Max_vocab,Max_length)
test_tokenized = tokenizer.texts_to_sequences(test_cleaned)
test_samples = keras.preprocessing.sequence.pad_sequences(test_tokenized,maxlen=Max_length) 
print(train_samples.shape,test_samples.shape,y_test.shape,y_test.shape)<jupyter_output>(77814, 50) (8646, 50) (8646,) (8646,)<jupyter_text>Defining the model.<jupyter_code>model = Sequential()

model.add(Embedding(Max_vocab,Embedding_dim,input_length= train_samples.shape[1]))
model.add(Conv1D(filters=32, kernel_size=3, padding = 'same',activation='relu'))
model.add(MaxPooling1D())
model.add(Bidirectional(LSTM(256,return_sequences = True,activation = 'relu')))
model.add(Dropout(0.6))
model.add(Bidirectional(LSTM(256,activation = 'relu')))
model.add(Dropout(0.3))
model.add(Dense(1024,activation = 'relu'))
model.add(Dropout(0.3))
model.add(Dense(512,activation = 'relu'))
model.add(Dropout(0.3))
model.add(Dense(1,activation = 'sigmoid'))

model.summary()<jupyter_output>Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding (Embedding)        (None, 50, 32)            320000    
_________________________________________________________________
conv1d (Conv1D)              (None, 50, 32)            3104      
_________________________________________________________________
max_pooling1d (MaxPooling1D) (None, 25, 32)            0         
_________________________________________________________________
bidirectional (Bidirectional (None, 25, 512)           591872    
_________________________________________________________________
dropout (Dropout)            (None, 25, 512)           0         
_________________________________________________________________
bidirectional_1 (Bidirection (None, 512)               1574912   
________________________________________________________[...]<jupyter_text>Compiling and training the model.<jupyter_code>model.compile(loss = BinaryCrossentropy(),optimizer=optimizers.Adam(),metrics=['accuracy'])

epochs= 10
batch_size=32
callbacks=EarlyStopping(monitor='val_loss',patience=3) 

history=model.fit(train_samples, y_train, 
                  epochs=epochs,
                  batch_size=batch_size,
                  validation_split=0.2,
                  callbacks=callbacks
                  )
model.save('model.h5')<jupyter_output><empty_output><jupyter_text>Predicting on train and test data.<jupyter_code>train_predictions = (model.predict(train_samples) > 0.5).astype("int32")
test_predictions = (model.predict(test_samples) > 0.5).astype("int32")<jupyter_output><empty_output><jupyter_text>Plotting training accuracy.<jupyter_code>plt.figure(figsize=[8,6])

plt.plot(history.history['accuracy'],'r',linewidth=3.0)

plt.plot(history.history['val_accuracy'],'b',linewidth=3.0)

plt.legend(['Training Accuracy', 'Validation Accuracy'],fontsize=18)

plt.xlabel('Epochs ',fontsize=16)

plt.ylabel('Accuracy',fontsize=16)

plt.title('Accuracy Curves',fontsize=16)<jupyter_output><empty_output><jupyter_text>Test Accuracy:<jupyter_code>print("Test Accuracy is : ",accuracy_score(y_test,test_predictions))<jupyter_output>Test Accuracy is :  0.7692574600971548<jupyter_text>Function for plotting confusion matrix(from sklearn documentation)<jupyter_code>def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):

    """

    This function prints and plots the confusion matrix.

    Normalization can be applied by setting `normalize=True`.

    """

    plt.figure(figsize=(10,10))



    plt.imshow(cm, interpolation='nearest', cmap=cmap)

    plt.title(title)

    plt.colorbar()



    tick_marks = np.arange(len(classes))

    plt.xticks(tick_marks, classes, rotation=45)

    plt.yticks(tick_marks, classes)



    if normalize:

        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

        cm = np.around(cm, decimals=2)

        cm[np.isnan(cm)] = 0.0

        print("Normalized confusion matrix")

    else:

        print('Confusion matrix, without normalization')

    thresh = cm.max() / 2.

    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):

        plt.text(j, i, cm[i, j],

                 horizontalalignment="center",

                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()

    plt.ylabel('True label')

    plt.xlabel('Predicted label')<jupyter_output><empty_output><jupyter_text>Plotting the confusion matrix.<jupyter_code>train_cm = confusion_matrix(y_train, train_predictions)
cm_plot_labels = ['class 0','class 1']
plot_confusion_matrix(cm=train_cm, classes=cm_plot_labels, title='Training Confusion Matrix')
test_cm = confusion_matrix(y_test, test_predictions)
cm_plot_labels = ['class 0','class 1']
plot_confusion_matrix(cm=test_cm, classes=cm_plot_labels, title='Test Confusion Matrix')<jupyter_output>Confusion matrix, without normalization<jupyter_text>Calculating Matthews Correlation Coefficient .<jupyter_code>train_mcc = matthews_corrcoef(y_train, train_predictions)
test_mcc = matthews_corrcoef(y_test, test_predictions)
print("Training Matthews Correlation Coefficient is : ",train_mcc)
print("Test Matthews Correlation Coefficient is : ",test_mcc)<jupyter_output>Training Matthews Correlation Coefficient is :  0.7934713307872834
Test Matthews Correlation Coefficient is :  0.5390040560925938
{'fraction_non_alphanumeric': 0.11812315432571366, 'fraction_numerical': 0.02679645630537023, 'mean_word_length': 5.839192221391174, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '647951', 'license': 'MIT', 'n_tokens_mistral': 3216, 'n_tokens_neox': 2947, 'n_words': 534, 'path': 'Task 1.ipynb', 'repo_name': 'gautam-girotra/computer-vision-assignment', 'size': '9143'}