<jupyter_start><jupyter_text>Preprocessing<jupyter_code>train["categories"] = train["categories"].astype(str)
train["converse"] = train["converse"].astype(str)
test["converse"] = test["converse"].astype(str)
print(train.shape)
print(test.shape)
print("Max length: %s" %train.converse.str.len().max())
print("Mean length: %s" %train.converse.str.len().mean())
print("Min length: %s" %train.converse.str.len().min())
MAX_SEQ_LEN = 450
VOCAB_SIZE = 20000
tokenizer = Tokenizer(num_words = VOCAB_SIZE)
tokenizer.fit_on_texts(train["converse"])

train_sequences = tokenizer.texts_to_sequences(train["converse"])
test_sequences = tokenizer.texts_to_sequences(test["converse"])
word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
train_data = pad_sequences(train_sequences, maxlen= MAX_SEQ_LEN)
test_data = pad_sequences(test_sequences, maxlen= MAX_SEQ_LEN)
print(train_data.shape)
print(test_data.shape)
print(len(np.unique(train_data)))
train_labels = train['categories']
label_encoder = LabelEncoder()
label_encoder.fit(train_labels)
train_labels = label_encoder.transform(train_labels)

print(label_encoder.classes_)
print(np.unique(train_labels, return_counts = True))
#convert to labels to one-hot-coded
train_tensor = to_categorical(np.array(train_labels))

print("Train tensor shape:",train_data.shape)
print("Train labels tensor shape:",train_tensor.shape)<jupyter_output>('Train tensor shape:', (48699, 450))
('Train labels tensor shape:', (48699, 21))<jupyter_text>Building Convolutional Neural Network<jupyter_code>EMBEDDING_DIM = 100
INITIALIZER = 'he_uniform'
REGULARIZER = regularizers.l2(0.01)

cnn_model = Sequential()
cnn_model.add(Embedding(input_dim= VOCAB_SIZE, 
                    output_dim= EMBEDDING_DIM ,
                    input_length= MAX_SEQ_LEN,
                    embeddings_regularizer = REGULARIZER
                   ))
cnn_model.add(Conv1D(filters= 64, kernel_size= 3,activation= 'relu',
                 kernel_initializer= INITIALIZER ,
                 #kernel_regularizer= REGULARIZER
                ))
cnn_model.add(Conv1D(filters= 64, kernel_size= 5,activation= 'relu',
                 kernel_initializer= INITIALIZER,
                 #kernel_regularizer= REGULARIZER
                 ))
cnn_model.add(GlobalMaxPool1D())
#cnn_model.add(MaxPool1D())
cnn_model.add(Dropout(0.2))
cnn_model.add(BatchNormalization())

#cnn_model.add(Conv1D(filters= 64, kernel_size= 5,activation= 'relu',padding = 'same'))
#cnn_model.add(Conv1D(filters= 64, kernel_size= 5,activation= 'relu',padding = 'same'))

cnn_model.add(Dense(128, activation= 'relu'))
cnn_model.add(Dropout(0.5))
cnn_model.add(BatchNormalization())
cnn_model.add(Dense(21,activation= 'softmax'))<jupyter_output><empty_output><jupyter_text>Configure the model for training<jupyter_code>adam = Adam(lr=0.001)
#adagrad = Adagrad()
#adadelta = Adadelta()
#nadam = Nadam(lr=0.001)
cnn_model.compile(loss = 'categorical_crossentropy',
             optimizer = adam,
             metrics = ['accuracy']
             )

cnn_model.summary()
metric = "val_acc"<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_3 (Embedding)      (None, 450, 100)          2000000   
_________________________________________________________________
conv1d_3 (Conv1D)            (None, 448, 64)           19264     
_________________________________________________________________
conv1d_4 (Conv1D)            (None, 444, 64)           20544     
_________________________________________________________________
global_max_pooling1d_2 (Glob (None, 64)                0         
_________________________________________________________________
dropout_5 (Dropout)          (None, 64)                0         
_________________________________________________________________
batch_normalization_5 (Batch (None, 64)                256       
_________________________________________________________________
dense_5 (D[...]<jupyter_text>Define callbacks<jupyter_code>reduce_lr = callbacks.ReduceLROnPlateau(monitor= metric, factor= 0.1,
                                       patience= 10, min_lr= 0.00001,
                                       verbose= 1)

early_stop = callbacks.EarlyStopping(monitor= metric, mode= 'auto',
                                     patience= 15,
                                    verbose= 1)
                                    #restore_best_weights= True)

#file_path = "cnn_weights.{epoch:02d}-{val_loss:.2f}.hdf5"
file_path = "../models/cnn_weights.hdf5"
model_chkpt = callbacks.ModelCheckpoint(filepath=file_path,monitor= metric, save_best_only=True)<jupyter_output><empty_output><jupyter_text>Fit the model<jupyter_code>history = cnn_model.fit(train_data , train_tensor ,
         batch_size = 64, shuffle = True,
         epochs=1000,validation_split = 0.2,
         callbacks = [reduce_lr,early_stop,model_chkpt]
         )<jupyter_output>Train on 38959 samples, validate on 9740 samples
Epoch 1/1000
38959/38959 [==============================] - 11s 293us/step - loss: 2.5941 - acc: 0.5524 - val_loss: 1.3293 - val_acc: 0.6992
Epoch 2/1000
38959/38959 [==============================] - 9s 220us/step - loss: 1.3677 - acc: 0.6801 - val_loss: 1.1958 - val_acc: 0.7144
Epoch 3/1000
38959/38959 [==============================] - 8s 217us/step - loss: 1.2640 - acc: 0.6987 - val_loss: 1.1256 - val_acc: 0.7180
Epoch 4/1000
38959/38959 [==============================] - 8s 217us/step - loss: 1.2157 - acc: 0.7041 - val_loss: 1.1215 - val_acc: 0.7249
Epoch 5/1000
38959/38959 [==============================] - 8s 215us/step - loss: 1.1900 - acc: 0.7103 - val_loss: 1.1184 - val_acc: 0.7155
Epoch 6/1000
38959/38959 [==============================] - 8s 216us/step - loss: 1.1726 - acc: 0.7152 - val_loss: 1.1199 - val_acc: 0.7215
Epoch 7/1000
38959/38959 [==============================] - 8s 218us/step - loss: 1.1551 - acc: 0.7189 - val_l[...]<jupyter_text>Capture the model metrics<jupyter_code>train_acc = history.history['acc']
train_loss = history.history['loss']

val_acc = history.history['val_acc']
val_loss = history.history['val_loss']<jupyter_output><empty_output><jupyter_text>Plot the Model accuracies and losses<jupyter_code>plt.figure(figsize=(20,5))
ax = plt.subplot(1,2,1)
ax.set_title("Model Loss")
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.plot(train_loss , label ="Train Loss")
plt.plot(val_loss, label ="Val Loss")
plt.legend()

ax = plt.subplot(1,2,2)
ax.set_title("Model Accuracy")
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.plot(train_acc, label = "Train Accuracy")
plt.plot(val_acc, label = "Val Accuracy")
plt.legend()
plt.show();
best_weights_cnn = load_model(file_path)
evaluation = best_weights_cnn.evaluate(train_data,train_tensor)
print("Train Loss: %f"%evaluation[0] , "Train Accuracy: %f"%evaluation[1])<jupyter_output>48699/48699 [==============================] - 3s 65us/step
('Train Loss: 0.696461', 'Train Accuracy: 0.798066')<jupyter_text>&nbsp; Building CuDNNLSTM model<jupyter_code>EMBEDDING_DIM = 100
INITIALIZER = 'he_uniform'
REGULARIZER = regularizers.l2(0.01)

cuDNN_LSTM_model = Sequential()
cuDNN_LSTM_model.add(Embedding(input_dim= VOCAB_SIZE, 
                    output_dim= EMBEDDING_DIM ,
                    input_length= MAX_SEQ_LEN,
                    embeddings_regularizer = REGULARIZER
                   ))
#cuDNN_LSTM_model.add(Lambda(lambda x: tf.expand_dims(cuDNN_LSTM_model.output, axis=-1)))
cuDNN_LSTM_model.add(CuDNNLSTM(128 ,#activation= 'relu',
               return_sequences=True,
               name="lstm_1"
              ))
cuDNN_LSTM_model.add(CuDNNLSTM(64 ,#activation= 'relu',
               return_sequences=False,
               name="lstm_2"
              ))
cuDNN_LSTM_model.add(Dropout(0.2))
cuDNN_LSTM_model.add(BatchNormalization())

cuDNN_LSTM_model.add(Dense(128,activation= 'relu'))
cuDNN_LSTM_model.add(Dropout(0.5))
cuDNN_LSTM_model.add(BatchNormalization())
cuDNN_LSTM_model.add(Dense(21,activation= 'softmax'))
adam = Adam(lr=0.001)
#adagrad = Adagrad()
#adadelta = Adadelta()
#nadam = Nadam(lr=0.001)
cuDNN_LSTM_model.compile(loss = 'categorical_crossentropy',
             optimizer = adam,
             metrics = ['accuracy']
             )

cuDNN_LSTM_model.summary()
reduce_lr = callbacks.ReduceLROnPlateau(monitor= metric, factor= 0.1,
                                       patience= 10, min_lr= 0.00001,
                                       verbose= 1)

early_stop = callbacks.EarlyStopping(monitor= metric, mode= 'auto',
                                     patience= 15,
                                    verbose= 1)
                                     #restore_best_weights= True)

#file_path = "cnn4_weights.{epoch:02d}-{val_loss:.2f}.hdf5"
file_path = "../models/cudnn_lstm_weights.hdf5"
model_chkpt = callbacks.ModelCheckpoint(filepath=file_path,monitor= metric, save_best_only=True)
history = cuDNN_LSTM_model.fit(train_data , train_tensor ,
         batch_size = 64, shuffle = True,
         epochs=1000,validation_split = 0.2,
         callbacks = [reduce_lr,early_stop,model_chkpt]
         )
train_acc = history.history['acc']
train_loss = history.history['loss']

val_acc = history.history['val_acc']
val_loss = history.history['val_loss']
plt.figure(figsize=(20,5))
ax = plt.subplot(1,2,1)
ax.set_title("Model Loss")
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.plot(train_loss , label ="Train Loss")
plt.plot(val_loss, label ="Val Loss")
plt.legend()

ax = plt.subplot(1,2,2)
ax.set_title("Model Accuracy")
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.plot(train_acc, label = "Train Accuracy")
plt.plot(val_acc, label = "Val Accuracy")
plt.legend()
plt.show();
best_weights_lstm = load_model(file_path)
evaluation = best_weights_lstm.evaluate(train_data,train_tensor)
print("Train Loss: %f"%evaluation[0] , "Train Accuracy: %f"%evaluation[1])<jupyter_output>48699/48699 [==============================] - 29s 586us/step
('Train Loss: 0.746703', 'Train Accuracy: 0.772788')<jupyter_text>&nbsp; Building CuDNNGRU model<jupyter_code>EMBEDDING_DIM = 100
INITIALIZER = 'he_uniform'
REGULARIZER = regularizers.l2(0.01)

cuDNN_GRU_model = Sequential()
cuDNN_GRU_model.add(Embedding(input_dim= VOCAB_SIZE, 
                    output_dim= EMBEDDING_DIM ,
                    input_length= MAX_SEQ_LEN,
                    embeddings_regularizer = REGULARIZER
                   ))


#cuDNN_GRU_model.add(Lambda(lambda x: tf.expand_dims(cuDNN_GRU_model.output, axis=-1)))
cuDNN_GRU_model.add(CuDNNGRU(128, return_sequences= True)) 
cuDNN_GRU_model.add(CuDNNGRU(64, return_sequences= True))
cuDNN_GRU_model.add(CuDNNGRU(32, return_sequences= False))
cuDNN_GRU_model.add(Dropout(0.2))
cuDNN_GRU_model.add(BatchNormalization())

cuDNN_GRU_model.add(Dense(128,activation= 'relu'))
cuDNN_GRU_model.add(Dropout(0.5))
cuDNN_GRU_model.add(BatchNormalization())
cuDNN_GRU_model.add(Dense(21,activation= 'softmax'))
adam = Adam(lr=0.001)
#adagrad = Adagrad()
#adadelta = Adadelta()
#nadam = Nadam(lr=0.001)
cuDNN_GRU_model.compile(loss = 'categorical_crossentropy',
             optimizer = adam,
             metrics = ['accuracy']
             )

cuDNN_GRU_model.summary()
reduce_lr = callbacks.ReduceLROnPlateau(monitor= metric, factor= 0.1,
                                       patience= 10, min_lr= 0.00001,
                                       verbose= 1)

early_stop = callbacks.EarlyStopping(monitor= metric, mode= 'auto',
                                     patience= 15,
                                    verbose= 1)
                                     #restore_best_weights= True)

#file_path = "cnn4_weights.{epoch:02d}-{val_loss:.2f}.hdf5"
file_path = "../models/cudnn_gru_weights.hdf5"
model_chkpt = callbacks.ModelCheckpoint(filepath=file_path,monitor= metric, save_best_only=True)
history = cuDNN_GRU_model.fit(train_data , train_tensor ,
         batch_size = 64, shuffle = True,
         epochs=1000,validation_split = 0.2,
         callbacks = [reduce_lr,early_stop,model_chkpt]
         )
train_acc = history.history['acc']
train_loss = history.history['loss']

val_acc = history.history['val_acc']
val_loss = history.history['val_loss']
plt.figure(figsize=(20,5))
ax = plt.subplot(1,2,1)
ax.set_title("Model Loss")
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.plot(train_loss , label ="Train Loss")
plt.plot(val_loss, label ="Val Loss")
plt.legend()

ax = plt.subplot(1,2,2)
ax.set_title("Model Accuracy")
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.plot(train_acc, label = "Train Accuracy")
plt.plot(val_acc, label = "Val Accuracy")
plt.legend()
plt.show();
best_weights_gru = load_model(file_path)
evaluation = best_weights_gru.evaluate(train_data,train_tensor)
print("Train Loss: %f"%evaluation[0] , "Train Accuracy: %f"%evaluation[1])
#plot_model(model, to_file='model.png')<jupyter_output><empty_output><jupyter_text>Write results to csv for submission<jupyter_code>test_preds = best_weights_gru.predict_classes(test_data)
result1_df = pd.DataFrame({"id" : test["id"]})
result1_df["categories"] = label_encoder.classes_[test_preds]
#print(result1_df.shape)
#result1_df.head()
result1_df.to_csv("../results/result3.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14981610748329957, 'fraction_numerical': 0.04773699617203333, 'mean_word_length': 3.995875515560555, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '376689', 'license': 'MIT', 'n_tokens_mistral': 5167, 'n_tokens_neox': 4585, 'n_words': 722, 'path': 'notebooks/NeuralNetworks_TextClassification.ipynb', 'repo_name': 'sadashiv9700/nn_text_classification', 'size': '13323'}