<jupyter_start><jupyter_text>Toxic Comment Classification  Load Packages<jupyter_code>import numpy as np
import pandas as pd
import tensorflow as tf

import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.pyplot as pylab
import seaborn as sns

import re
import keras

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, GRU
from keras.layers import Bidirectional, GlobalMaxPool1D, Conv1D, GlobalAveragePooling1D, concatenate
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.callbacks import CSVLogger, ReduceLROnPlateau, ModelCheckpoint 
from keras.models import Sequential<jupyter_output>Using TensorFlow backend.<jupyter_text>Load Data & Basic EDA<jupyter_code># Word Embeddings
glove_path = 'datasets/glove.6B.50d.txt'
# List Classfier
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
# Read Train Data
train_df = pd.read_csv("datasets/train.csv")
train_df.dropna(inplace=True)
train_df.head(10)
# Read Test Data
test_df = pd.read_csv("datasets/test.csv")
test_df.dropna(inplace=True)
test_df.head(10)
# Read Test Label Data
test_labels_df = pd.read_csv("datasets/test_labels.csv")
test_labels_df.dropna(inplace=True)
test_labels_df.head(10)
#(pd.isnull(train_df).sum())
train_df.count()
# Double check if test_df and test_labels_df got the same number of rows of dataset
test_df["id"].count() == test_labels_df["id"].count()
# Check a comment BEFORE standardising the contractions and removal of stopwords
train_df['comment_text'][77]
# Standardise contractions to standard english word
# Removal of stopwords
# eg: can't OR cant --> cannot
for dataset in [train_df, test_df]:
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('\'ll', ' will'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('\'ve', ' have'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('don\'t', ' do not'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('dont', ' do not'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('aren\'t', ' are not'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('won\'t', ' will not'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('wont', ' will not'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('can\'t', ' cannot'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('cant', ' cannot'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('shan\'t', ' shall not'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('shant', ' shall not'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace('\'m', ' am'))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace("doesn't", "does not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace("doesnt", "does not"))                                                      
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace( "didn't", "did not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace( "didnt", "did not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace("hasn't", "has not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace("hasnt", "has not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace("haven't", "have not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace("havent", "have not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace("wouldn't", "would not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace( "didn't", "did not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace( "didnt", "did not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace("it's" , "it is"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace( "that's" , "that is"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace("weren't" , "were not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace("werent" , "were not"))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace(' u ', ' you '))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: x.replace(' U ', ' you '))
    dataset['comment_text'] = dataset['comment_text'].apply(lambda x: re.sub('[\(\)\"\t_\n.,:=!@#$%^&*-/[\]?|1234567890—]', ' ', x).strip())
# Check a comment AFTER standardising the contractions and removal of stopwords
train_df['comment_text'][77]
# Heatmap to see the likeliness of a comment falling into 2 categories
plt.figure(figsize=(10,10))
plt.title('Likeliness of a comment in 2 categories',y=1.05,size=20)
sns.heatmap(train_df[list_classes].astype(float).corr(),linewidths=0.5,vmax=1.0,square=True,annot=True)
plt.show()
MAX_FEATURES = 20000
MAX_LENGTH = 100
EMBED_SIZE = 50
BATCH_SIZE = 32
EPOCHS = 5<jupyter_output><empty_output><jupyter_text>Tokenizing data<jupyter_code>tokenizer = Tokenizer(num_words=MAX_FEATURES)
tokenizer.fit_on_texts(train_df['comment_text'])
train_tokenized = tokenizer.texts_to_sequences(train_df['comment_text'])
test_tokenized = tokenizer.texts_to_sequences(test_df['comment_text'])
X_train = pad_sequences(train_tokenized, maxlen=MAX_LENGTH)
X_test = pad_sequences(test_tokenized, maxlen=MAX_LENGTH)
Y = train_df[list_classes]<jupyter_output><empty_output><jupyter_text>Embedding Matrix<jupyter_code>embedding_index = {}
with open(glove_path, encoding='utf8') as f:
    for line in f:
        values = line.rstrip().rsplit(' ')
        embedding_index[values[0]] = np.asarray(values[1:], dtype='float32')

word_index = tokenizer.word_index
num_words = min(MAX_FEATURES, len(word_index) + 1)
embedding_matrix = np.zeros((num_words, EMBED_SIZE))
for word, i in word_index.items():
    if i >= MAX_FEATURES:
        continue

    embedding_vector = embedding_index.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector<jupyter_output><empty_output><jupyter_text>Build the model<jupyter_code>model = Sequential([
    Embedding(MAX_FEATURES, EMBED_SIZE),
    Bidirectional(LSTM(64,return_sequences=True,dropout=0.1,recurrent_dropout=0.1)),
    GlobalMaxPool1D(),
    Dropout(0.1),
    Dense(64,activation='relu'),
    Dropout(0.1),
    Dense(6,activation="sigmoid")
])
model.summary()
csv_logger = CSVLogger('log.csv', append=True, separator=',')
checkpoint = ModelCheckpoint('checkpoint.csv', monitor='val_acc', verbose=1, save_best_only=True, mode='max')
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=1, min_lr=1e-8)
model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])
models = model.fit(X_train, Y, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.1,
                   callbacks=[csv_logger, reduce_lr, checkpoint])
#model.load_weights('checkpoint.csv')
sub = model.predict(X_test)
temp = pd.read_csv('datasets/sample_submission.csv')
temp[list_classes] = sub
temp.to_csv('datasets/submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15766710353866317, 'fraction_numerical': 0.010747051114023591, 'mean_word_length': 6.365830115830116, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '624849', 'license': 'MIT', 'n_tokens_mistral': 2605, 'n_tokens_neox': 2515, 'n_words': 502, 'path': 'Toxic Comment Classification.ipynb', 'repo_name': 'nicholasneo78/toxic-comment-classifier', 'size': '7630'}