<jupyter_start><jupyter_text>-------------------------------- Training model  -----------------------------------  Model Paremeters<jupyter_code>model_type    = 'CNN-non-static'  # CNN-rand|CNN-non-static|CNN-static
embedding_dim = 100         # word2vec dim
vocab_size    = dictionary_size
max_sent_len  = seq_len
N_CATEGORY    = 5<jupyter_output><empty_output><jupyter_text>Generate word2vec<jupyter_code>if model_type in ['CNN-non-static', 'CNN-static']:
    embedding_wts = train_word2vec( np.vstack((x_train, x_test)), 
                                    dictionary.token2id, num_features = embedding_dim)
    if model_type == 'CNN-static':
        x_train = embedding_wts[0][x_train]
        x_test  = embedding_wts[0][x_test]
        
elif model_type == 'CNN-rand':
    embedding_wts = None
    
else:
    raise ValueError("Unknown model type")<jupyter_output>Load existing Word2Vec model '100features_1minwords_10context'<jupyter_text>Create model<jupyter_code>filter_sizes = [3,4,5]
num_filters  = 50
dropout_prob = (0.5, 0.8)
hidden_dims  = 50
batch_size   = 64
l2_reg = 0.3

# Deciding dimension of input based on the model
input_shape = (max_sent_len, embedding_dim) if model_type == "CNN-static" else (max_sent_len,)
model_input = Input(shape = input_shape)

# Static model do not have embedding layer
if model_type == "CNN-static":
    z = Dropout(dropout_prob[0])(model_input)
else:
    z = Embedding(vocab_size, embedding_dim, input_length = max_sent_len, name="embedding", trainable=True)(model_input)
    z = Dropout(dropout_prob[0])(z)

# Convolution layers
z1 = Conv1D(    filters=num_filters, kernel_size=filter_sizes[0], 
                padding="valid", activation="relu", 
                strides=1)(z)
z1 = MaxPooling1D(pool_size = int(max_sent_len - filter_sizes[0] + 1))(z1)
z1 = Flatten()(z1)

z2 = Conv1D(    filters=num_filters, kernel_size=filter_sizes[1], 
                padding="valid", activation="relu", 
                strides=1)(z)
z2 = MaxPooling1D(pool_size= int(max_sent_len - filter_sizes[1] + 1))(z2)
z2 = Flatten()(z2)

z3 = Conv1D(    filters=num_filters, kernel_size=filter_sizes[2], 
                padding="valid", activation="relu",
                strides=1)(z)
z3 = MaxPooling1D(pool_size= int(max_sent_len - filter_sizes[2] + 1))(z3)
z3 = Flatten()(z3)

# Concatenate the output of all convolution layers
z = Concatenate()([z1, z2, z3])
z = Dropout(dropout_prob[1])(z)

z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense( N_CATEGORY, activation="softmax")(z)
    
model = Model(model_input, model_output)
model.summary()<jupyter_output>____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
input_1 (InputLayer)             (None, 21)            0                                            
____________________________________________________________________________________________________
embedding (Embedding)            (None, 21, 100)       1380100                                      
____________________________________________________________________________________________________
dropout_1 (Dropout)              (None, 21, 100)       0                                            
____________________________________________________________________________________________________
conv1d_1 (Conv1D)                (None, 19, 50)        15050                               [...]<jupyter_text>Train model<jupyter_code>from sklearn.metrics import roc_auc_score
import time
import pylab as pl
from IPython import display
pl.style.use('ggplot')
%matplotlib inline

class Histories(keras.callbacks.Callback):

    def on_train_begin(self, logs={}):
        self.acc = []
        self.loss = []
        self.val_loss = []
        self.val_acc = []

    def on_train_end(self, logs={}):
        return

    def on_epoch_begin(self, epoch, logs={}):
        return

    def on_epoch_end(self, epoch, logs={}):
        self.acc.append(logs['acc'])
        self.loss.append(logs['loss'])
        self.val_acc.append(logs['val_acc'])
        self.val_loss.append(logs['val_loss'])
        
        pl.hold(True)
        pl.plot(self.acc)
        pl.plot(self.loss)
        pl.plot(self.val_acc)
        pl.plot(self.val_loss)
        pl.legend(['Train acc','Train loss','Valid acc', 'Valid loss'], loc=2)
        display.clear_output(wait=True)
        display.display(pl.gcf())
        return

    def on_batch_begin(self, batch, logs={}):
        return

    def on_batch_end(self, batch, logs={}):
        return
if model_type == "CNN-non-static":
    embedding_layer = model.get_layer("embedding")
    embedding_layer.set_weights(embedding_wts)

model.compile(loss="categorical_crossentropy", optimizer=optimizers.SGD(), metrics=["accuracy"])

histories = Histories()
res = model.fit(x_train, y_train, 
          batch_size = batch_size,
          epochs=200,
          validation_data=(x_test, y_test), verbose=2, callbacks=[histories])
import os
os.system('say done')<jupyter_output><empty_output><jupyter_text>Computing Accuracy<jupyter_code>scores = model.evaluate(x_train, y_train, verbose=0)
print("Train Accuracy: %.2f%%" % (scores[1]*100))
scores = model.evaluate(x_test, y_test, verbose=0)
print("Test Accuracy: %.2f%%" % (scores[1]*100))<jupyter_output>Train Accuracy: 38.80%
Test Accuracy: 33.62%<jupyter_text>Save the model<jupyter_code>date = str(datetime.date.today() )
time = str(datetime.datetime.now().time())[:-7]

filename = './newtrain_' + model_type + '_' + date + '_' +time

with open( filename + '_history', 'wb') as output:
    pickle.dump([model.history.history], output, pickle.HIGHEST_PROTOCOL)
    
model.save(filename + '.h5')<jupyter_output><empty_output><jupyter_text>Plot the results<jupyter_code>import matplotlib.pyplot as plt
plt.style.use('ggplot')

plt.rcParams["figure.figsize"] = (10,6)
plt.plot(model.history.history['acc'])
plt.plot(model.history.history['loss'])
plt.plot(model.history.history['val_acc'])
plt.plot(model.history.history['val_loss'])
plt.legend(['Train acc','Train loss','Valid acc', 'Valid loss'], loc=2)
plt.xlabel('Epochs')
plt.ylabel('Loss/Accuracy')
plt.title('Using '+ model_type)
# imgName = 'Images/' + model_type + '_' + date + '_' + time + '.jpg'
# plt.savefig( imgName, dpi= 200, bbox_inches='tight', transparent=False)
plt.show()<jupyter_output><empty_output><jupyter_text>Continue from a pretrained model<jupyter_code>import pickle

# filename = './newtrain_CNN-rand_2017-04-08_16:29:11'
model = keras.models.load_model(filename)
model.compile(loss="categorical_crossentropy", optimizer=optimizers.SGD(0.005), metrics=["accuracy"])

histories = Histories()
res = model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=110,
          validation_data=(x_test, y_test), verbose=2, callbacks=[histories])<jupyter_output>Train on 156060 samples, validate on 2210 samples
Epoch 1/300
27s - loss: 0.8415 - acc: 0.6568 - val_loss: 1.4342 - val_acc: 0.4005
Epoch 2/300
27s - loss: 0.8416 - acc: 0.6565 - val_loss: 1.4266 - val_acc: 0.4005
Epoch 3/300
24s - loss: 0.8389 - acc: 0.6578 - val_loss: 1.4288 - val_acc: 0.4014
Epoch 4/300
26s - loss: 0.8403 - acc: 0.6564 - val_loss: 1.4244 - val_acc: 0.4041
Epoch 5/300
25s - loss: 0.8391 - acc: 0.6589 - val_loss: 1.4318 - val_acc: 0.4009
Epoch 6/300
25s - loss: 0.8393 - acc: 0.6585 - val_loss: 1.4242 - val_acc: 0.4023
Epoch 7/300
25s - loss: 0.8373 - acc: 0.6582 - val_loss: 1.4232 - val_acc: 0.4023
Epoch 8/300
25s - loss: 0.8398 - acc: 0.6571 - val_loss: 1.4261 - val_acc: 0.3986
Epoch 9/300
25s - loss: 0.8365 - acc: 0.6595 - val_loss: 1.4225 - val_acc: 0.4014
Epoch 10/300
26s - loss: 0.8355 - acc: 0.6593 - val_loss: 1.4259 - val_acc: 0.4063
Epoch 11/300
30s - loss: 0.8350 - acc: 0.6591 - val_loss: 1.4273 - val_acc: 0.4072
Epoch 12/300
26s - loss: 0.8342 - acc: 0.6587 [...]<jupyter_text>Save the model (marking as continued)<jupyter_code>date = str(datetime.date.today() )
time = str(datetime.datetime.now().time())[:-7]

filename = './newtrain_continued_' + model_type + '_' + date + '_' +time

with open( filename + '_history', 'wb') as output:
    pickle.dump([model.history.history], output, pickle.HIGHEST_PROTOCOL)
    
model.save(filename + '.h5';)
import os
os.system('say done')
files = [
    'CNN-rand_continued_2017-04-02_19:26:35',
    'CNN-rand_continued_2017-04-03_16:50:47',
    'CNN-rand_continued_2017-04-03_17:18:47'
]
train_acc, train_loss, val_acc, val_loss = [],[],[],[]

for file in files:
    hist = out[0][0]
    train_acc += hist.history['acc']
    train_loss += hist.history['loss']
    val_acc += hist.history['val_acc']
    val_loss += hist.history['val_loss']
import matplotlib.pyplot as plt
plt.style.use('ggplot')
plt.rcParams["figure.figsize"] = (10,6)
plt.plot(train_acc)
plt.plot(train_loss)
plt.plot(val_acc)
# plt.plot(val_loss)
plt.legend(['Train acc','Train loss','Valid acc', 'Valid loss'], loc=2)
plt.xlabel('Epochs')
plt.ylabel('Loss/Accuracy')
plt.title('Using '+ model_type)
date = str(datetime.date.today() )
time = str(datetime.datetime.now().time())[:-7]
imgName = 'Images/' + model_type + '_' + date + '_' + time + '.jpg'
plt.savefig( imgName, dpi= 200, bbox_inches='tight', transparent=False)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1565079365079365, 'fraction_numerical': 0.056296296296296296, 'mean_word_length': 3.653372722796652, 'pattern_counts': {'":': 2, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '107301', 'license': 'MIT', 'n_tokens_mistral': 3684, 'n_tokens_neox': 3288, 'n_words': 624, 'path': 'PRNN/Assignment/Assignment3/.ipynb_checkpoints/CNN_SST-checkpoint.ipynb', 'repo_name': 'ashutoshml/Assignments', 'size': '9450'}