<jupyter_start><jupyter_text>Upgrade I<jupyter_code>#HW1 model, please revise the model
import keras
from keras.utils import np_utils
from keras.models import Sequential 
from keras.layers import Dense, Activation 
from keras import initializers, optimizers
import time

#define variables
input_dim = 784 
nb_classes = 10
output_dim = nb_classes = 10 
batch_size = 64 
epoch = 50
l_r = 0.01
rho = 0.99
variance_scaling = initializers.VarianceScaling()

#Transform Y's to categorical
Y_train = np_utils.to_categorical(Y_train, nb_classes) 
Y_test = np_utils.to_categorical(Y_test, nb_classes)
Y_val = np_utils.to_categorical(Y_val, nb_classes)
print(X_train.shape,Y_train.shape)

def basic_model():
    model = Sequential() 
    model.add(Dense(64,input_dim=input_dim, kernel_initializer=variance_scaling, bias_initializer='zeros', activation='relu'))
    model.add(Dense(10, kernel_initializer=variance_scaling, bias_initializer='zeros', activation="softmax"))
    return model
def run_model(model, X_train, X_val, X_test):
    rmsprop = optimizers.RMSprop(lr=l_r, rho=rho)
    model.compile(optimizer=rmsprop, loss='categorical_crossentropy', metrics=['accuracy']) 
    start_time = time.time()
    model.fit(X_train, Y_train, batch_size=batch_size, epochs=epoch,verbose=1, validation_data=(X_val, Y_val)) 
    end_time = time.time()
    print(model.summary())
    print('Run Time for Training:', end_time-start_time)
    score = model.evaluate(X_test, Y_test) 
    print('Test accuracy:', score[1])
basic_model = basic_model()
run_model(basic_model, X_train, X_val, X_test)<jupyter_output>(50000, 784) (50000, 10)
Train on 50000 samples, validate on 10000 samples
Epoch 1/50
50000/50000 [==============================] - 3s 58us/step - loss: 0.3963 - acc: 0.9103 - val_loss: 0.1555 - val_acc: 0.9524
Epoch 2/50
50000/50000 [==============================] - 2s 38us/step - loss: 0.1613 - acc: 0.9529 - val_loss: 0.1239 - val_acc: 0.9644
Epoch 3/50
50000/50000 [==============================] - 2s 41us/step - loss: 0.1358 - acc: 0.9606 - val_loss: 0.1409 - val_acc: 0.9632
Epoch 4/50
50000/50000 [==============================] - 2s 43us/step - loss: 0.1203 - acc: 0.9649 - val_loss: 0.2684 - val_acc: 0.9280
Epoch 5/50
50000/50000 [==============================] - 2s 40us/step - loss: 0.1082 - acc: 0.9681 - val_loss: 0.2471 - val_acc: 0.9369
Epoch 6/50
50000/50000 [==============================] - 2s 41us/step - loss: 0.1004 - acc: 0.9710 - val_loss: 0.4286 - val_acc: 0.9079
Epoch 7/50
50000/50000 [==============================] - 2s 42us/step - loss: 0.0935 - acc: 0.9730 - v[...]<jupyter_text>* Test Accuracy: 0.9665* Run Time: 104.376* Number of Epochs: 50* Batch Size: 64  Upgrade II<jupyter_code>from keras.layers import Dropout, Flatten, Conv2D, MaxPooling2D
from matplotlib import pyplot as plt
%matplotlib inline

#Reshape the x-train data
X_train = np.reshape(X_train, ([X_train.shape[0],28,28,1]))
print(X_train.shape)
X_val = np.reshape(X_val, ([X_val.shape[0],28,28,1]))
X_test = np.reshape(X_test, ([X_test.shape[0],28,28,1]))

#redefine variables
epoch = 5
l_r = 0.001

# plt.imshow(X_train[100].squeeze())

# define the CNN model
def cnn_model():
    model = Sequential()
    model.add(Conv2D(32, (5,5), activation='relu', input_shape=(28,28,1), kernel_initializer=variance_scaling))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Dropout(0.20))
    model.add(Flatten())
    model.add(Dense(128, kernel_initializer=variance_scaling, bias_initializer='zeros', activation='relu'))
    model.add(Dense(10, kernel_initializer=variance_scaling, bias_initializer='zeros', activation='softmax'))
    return model


cnn_model = cnn_model()
run_model(cnn_model, X_train, X_val, X_test)<jupyter_output>(50000, 28, 28, 1)
Train on 50000 samples, validate on 10000 samples
Epoch 1/5
50000/50000 [==============================] - 21s 418us/step - loss: 0.1608 - acc: 0.9524 - val_loss: 0.0718 - val_acc: 0.9802
Epoch 2/5
50000/50000 [==============================] - 20s 403us/step - loss: 0.0575 - acc: 0.9816 - val_loss: 0.0494 - val_acc: 0.9853
Epoch 3/5
50000/50000 [==============================] - 20s 407us/step - loss: 0.0383 - acc: 0.9881 - val_loss: 0.0449 - val_acc: 0.9867
Epoch 4/5
50000/50000 [==============================] - 22s 430us/step - loss: 0.0258 - acc: 0.9916 - val_loss: 0.0411 - val_acc: 0.9886
Epoch 5/5
50000/50000 [==============================] - 21s 416us/step - loss: 0.0197 - acc: 0.9937 - val_loss: 0.0552 - val_acc: 0.9844
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_30 (Conv2D)           (None, 24, 24,[...]<jupyter_text>* Test Accuracy: 0.985* Run Time: 103.9* Number of Epochs: 5* Batch Size: 64* Learning Rate: 0.001  Questions1) What is the dimension of the output of each neuron of the first layer for a batch of images (size 28× 28× N)?    Answer: 24*242) If an image I has dimension (width, height, depth) of W × H × D and is passed through a convolutional layer with N filters of filter size F, stride S and zero padding Z, what is the dimension of the filtered output for I?    Answer: ((W+2*Z-F)/S) + 1, ((H+2*Z-F)/S) + 1, F3) In this model, what is the dimension of the output of each neuron of the second layer(Maxpooling) for a batch of images (size 28× 28× N)?    Answer: 12*12<jupyter_code>dropout = 0.20
epoch = 15

def my_model():
    model = Sequential()
    model.add(Conv2D(32, (5,5), activation='relu', input_shape=(28,28,1), kernel_initializer=variance_scaling))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Dropout(dropout))

    model.add(Conv2D(32, (3,3), activation='relu', kernel_initializer=variance_scaling))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Dropout(dropout))

    model.add(Conv2D(32, (3,3), activation='relu', kernel_initializer=variance_scaling))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Dropout(dropout))

    model.add(Flatten())
    model.add(Dense(128, kernel_initializer=variance_scaling, bias_initializer='zeros', activation='relu'))
    model.add(Dropout(dropout))
    model.add(Dense(10, kernel_initializer=variance_scaling, bias_initializer='zeros', activation='softmax'))

    return model
my_model = my_model()
run_model(my_model, X_train, X_val, X_test)<jupyter_output>Train on 50000 samples, validate on 10000 samples
Epoch 1/15
50000/50000 [==============================] - 33s 666us/step - loss: 0.5615 - acc: 0.8149 - val_loss: 0.1186 - val_acc: 0.9619
Epoch 2/15
50000/50000 [==============================] - 31s 612us/step - loss: 0.2314 - acc: 0.9281 - val_loss: 0.0813 - val_acc: 0.9753
Epoch 3/15
50000/50000 [==============================] - 30s 605us/step - loss: 0.1714 - acc: 0.9480 - val_loss: 0.0645 - val_acc: 0.9812
Epoch 4/15
50000/50000 [==============================] - 31s 617us/step - loss: 0.1436 - acc: 0.9566 - val_loss: 0.0578 - val_acc: 0.9828
Epoch 5/15
50000/50000 [==============================] - 31s 630us/step - loss: 0.1244 - acc: 0.9625 - val_loss: 0.0548 - val_acc: 0.9844
Epoch 6/15
50000/50000 [==============================] - 31s 621us/step - loss: 0.1123 - acc: 0.9665 - val_loss: 0.0417 - val_acc: 0.9874
Epoch 7/15
50000/50000 [==============================] - 31s 616us/step - loss: 0.1031 - acc: 0.9686 - val_loss: 0.[...]
{'fraction_non_alphanumeric': 0.21179805615550756, 'fraction_numerical': 0.12311015118790497, 'mean_word_length': 4.668706962509564, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '252814', 'license': 'MIT', 'n_tokens_mistral': 3354, 'n_tokens_neox': 2693, 'n_words': 643, 'path': '18739-hws/hw2/dfreilic_hw2_part1.ipynb', 'repo_name': 'dfreilich/machine-learning-workspace', 'size': '7408'}