<jupyter_start><jupyter_text>Load Packages<jupyter_code>import numpy as np
import pandas as pd
import tensorflow as tf
import re
import keras

#import matplotlib as mpl
import matplotlib.pyplot as plt
#import matplotlib.pyplot as pylab
#import seaborn as sns
#import plotly.offline as py
#import plotly.graph_objs as go
#py.init_notebook_mode(connected=True)

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, GRU, Flatten
from keras.layers import Bidirectional, GlobalMaxPool1D, Conv1D, GlobalAveragePooling1D, concatenate
from keras.layers.embeddings import Embedding
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.callbacks import CSVLogger, ReduceLROnPlateau, ModelCheckpoint 

import pickle
#import nltk
#import string
#from nltk.corpus import stopwords
#from sklearn.manifold import TSNE

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer

#from sklearn.metrics import f1_score, roc_auc_score, multilabel_confusion_matrix, average_precision_score, precision_recall_curve<jupyter_output><empty_output><jupyter_text>Load Dataset and Basic EDA<jupyter_code># List Classifier
train_labels = ['outwear','top','trousers','women dresses','women skirts']
# Reading csv file of train set
fashion_df = pd.read_csv("./datasets/TIL_NLP_train_dataset.csv")
fashion_df.dropna(inplace=True)
fashion_df.head(10)
# Reading csv file of test set
fashion_test_df = pd.read_csv("./datasets/TIL_NLP_test_dataset.csv")
fashion_test_df.dropna(inplace=True)
fashion_test_df.head(10)
# Get the sentence with the most number of words and count how many words are there in the sentence
fashion_df_numpy = np.asarray(fashion_df)

# -----  DEBUG  -----
#fashion_df_numpy[0]
#fashion_df_numpy[2][1] 
# -----  DEBUG END -----

max_len = 0 # initialise 0 to the maximum number of words in a sentence
for i,j in enumerate(fashion_df_numpy):
    string = fashion_df_numpy[i][1]
    substring = 'w'
    count = string.count(substring)
    if count > max_len:
        max_len = count
print('Most number of words in a sentence: {}'.format(max_len))<jupyter_output>Most number of words in a sentence: 47<jupyter_text>Hence we can set the max length parameter to 50 (a number near 47) for pad_sequences for the Tokenizing portion of the EDA<jupyter_code>#This displays the relevant information of the dataframe
#Train Set
print("Information of Train Set\n")
fashion_df.info()
#Test Set
print("\nInformation of Test Set\n")
fashion_test_df.info()<jupyter_output>Information of Train Set

<class 'pandas.core.frame.DataFrame'>
Int64Index: 7380 entries, 0 to 7379
Data columns (total 7 columns):
 #   Column               Non-Null Count  Dtype 
---  ------               --------------  ----- 
 0   id                   7380 non-null   int64 
 1   word_representation  7380 non-null   object
 2   outwear              7380 non-null   int64 
 3   top                  7380 non-null   int64 
 4   trousers             7380 non-null   int64 
 5   women dresses        7380 non-null   int64 
 6   women skirts         7380 non-null   int64 
dtypes: int64(6), object(1)
memory usage: 461.2+ KB

Information of Test Set

<class 'pandas.core.frame.DataFrame'>
Int64Index: 2460 entries, 0 to 2459
Data columns (total 2 columns):
 #   Column               Non-Null Count  Dtype 
---  ------               --------------  ----- 
 0   id                   2460 non-null   int64 
 1   word_representation  2460 non-null   object
dtypes: int64(1), object(1)
memory usage: 57.7+[...]<jupyter_text>Explore pickle file<jupyter_code># Load the dictionary back from the pickle file.
words = pickle.load( open( "./datasets/word_embeddings.pkl", "rb" ) )
print("Printing out random mask: ",words["w7777"])
# Chec the vocabulary size of the dataset
vocabulary_size = len(words)
print("The vocabulary size is: {}".format(vocabulary_size))<jupyter_output>The vocabulary size is: 400000<jupyter_text>Create a bag of words<jupyter_code>docs = map(''.join, fashion_df[['word_representation']].values.tolist())

#instantiate CountVectorizer()
cv=CountVectorizer(stop_words="english", analyzer='word', 
                             ngram_range=(1, 2), max_df=1.0, min_df=1, max_features=None)
 
# this steps generates word counts for the words in your docs
word_count_vector=cv.fit_transform(docs)
word_count_vector.shape<jupyter_output><empty_output><jupyter_text>Use IDF & TF-IDF to check how relevant the 'masked' word is in a given document<jupyter_code># Fit the TF-IDF transformer
tfidf_transformer=TfidfTransformer(smooth_idf=True,use_idf=True)
tfidf_transformer.fit(word_count_vector)
# print idf values
df_idf = pd.DataFrame(tfidf_transformer.idf_, index=cv.get_feature_names(),columns=['idf'])
 
# sort ascending
df_idf = df_idf.sort_values(by=['idf'])
df_idf.head(10)
# count matrix
docs = map(''.join, fashion_df[['word_representation']].values.tolist())
count_vector=cv.transform(docs)

# tf-idf scores
tf_idf_vector=tfidf_transformer.transform(count_vector)

feature_names = cv.get_feature_names()
 
#get tfidf vector for first document
first_document_vector=tf_idf_vector[0]
 
#print the scores
df = pd.DataFrame(first_document_vector.T.todense(), index=feature_names, columns=["tfidf"])
df = df.sort_values(by=["tfidf"],ascending=False)
df.head(15)<jupyter_output><empty_output><jupyter_text>Tokenizing data<jupyter_code># CONSTANTS FOR TOKENIZING AND TRAINING OF MODEL
MAX_FEATURES = 20000
MAX_LENGTH = 50
EMBED_SIZE = 50
# CREATE A TOKENIZER INSTANCE
tokenizer = Tokenizer(num_words=MAX_FEATURES)
tokenizer.fit_on_texts(fashion_df['word_representation'])
# TOKENIZED THE TRAIN AND TEST SET FEATURES
train_tokenized = tokenizer.texts_to_sequences(fashion_df['word_representation'])
test_tokenized = tokenizer.texts_to_sequences(fashion_test_df['word_representation'])
# PAD SEQUENCES TO THE SAME LENGTH FOR THE FEATURES (SENTENCES/WORD REPRESENTATION)
X_train = pad_sequences(train_tokenized, maxlen=MAX_LENGTH)
X_test = pad_sequences(test_tokenized, maxlen=MAX_LENGTH)

# TRAIN LABELS
y_train = fashion_df[train_labels]<jupyter_output><empty_output><jupyter_text>Build the model<jupyter_code># DECLARE MODEL
model = Sequential([
    Embedding(MAX_FEATURES, EMBED_SIZE),
    Bidirectional(GRU(128,return_sequences=True,dropout=0.2,recurrent_dropout=0.05)),
    #GRU(150,return_sequences=True,dropout=0.1,recurrent_dropout=0.1),
    GlobalMaxPool1D(),
    Dropout(0.1),
    Dense(64,activation='relu'),
    Dropout(0.1),
    Dense(5,activation="sigmoid")
])
model.summary()
# Note down the useful informations into a .csv file
csv_logger = CSVLogger('./datasets/log.csv', append=True, separator=',')
checkpoint = ModelCheckpoint('./datasets/checkpoint.csv', monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=1, min_lr=1e-8)
model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])
# HYPERPARAMETERS CONSTANT
BATCH_SIZE = 16
EPOCHS = 4
VALIDATION_SPLIT = 0.1
models = model.fit(X_train, y_train, 
                   batch_size=BATCH_SIZE, 
                   epochs=EPOCHS, 
                   validation_split=VALIDATION_SPLIT,
                   callbacks=[csv_logger, reduce_lr, checkpoint])
#Plot the loss graph of the model
plt.title('Loss')
plt.plot(models.history['loss'], label='train')
plt.plot(models.history['val_loss'], label='validation')
plt.legend()
plt.show()
#Plot the accuracy graph of the model
plt.title('Accuracy')
plt.plot(models.history['accuracy'], label='train')
plt.plot(models.history['val_accuracy'], label='validation')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Predict the value of y (yhat)<jupyter_code>yhat_raw = model.predict(X_test, verbose=0)

# CHECK THE NUMPY ARRAY
yhat_raw
# DEBUG to peek at some values
#yhat_raw[777]
#yhat_raw[77]
# CONVERT PREDICTIONS INTO DISCRETE CLASSIFIER (0 OR 1)
# TOLERANCE CONSTANT
TOLERANCE = 0.5

# predicting crisp classes for test set
yhat = [[1 if x > TOLERANCE else 0 for idx,x in enumerate(i)] for i in yhat_raw]
yhat
# DEBUG to peek at some values
#yhat[777]
#yhat[77]
# CONVERT THE YHAT VALUES INTO A PANDAS DATAFRAME
yhat_df = pd.DataFrame(yhat,index=fashion_test_df['id'],columns=train_labels)
yhat_df<jupyter_output><empty_output><jupyter_text>Export the predictions as a .csv file<jupyter_code># INSERT FILE NAME
FILENAME = 'TEAM_MIA_nlp_predictions'
# EXPORT THE CSV FILE
yhat_df.to_csv('./datasets/{}.csv'.format(FILENAME), index=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10808646917534027, 'fraction_numerical': 0.022646688779595105, 'mean_word_length': 5.0470262793914245, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '644037', 'license': 'MIT', 'n_tokens_mistral': 3215, 'n_tokens_neox': 2960, 'n_words': 722, 'path': 'NLP-v3.ipynb', 'repo_name': 'FremontTeng/Team-MIA-TIL-2020', 'size': '8743'}