<jupyter_start><jupyter_text>Homework 6<NAME><jupyter_code>import numpy as np
import tensorflow as tf
from sklearn.preprocessing import OneHotEncoder
from sklearn.metrics import accuracy_score
# load MNIST data
MNIST = np.load('./Data/MNIST_train_1000.npz')
images_train = MNIST['train_images']
labels_train = MNIST['train_labels']
print('dimensions:')
print('training input:', images_train.shape, ' output:', labels_train.shape)
MNIST = np.load('./Data/MNIST_test_1000.npz')
images_test = MNIST['test_images']
labels_test = MNIST['test_labels']
print('testing input: ', images_test.shape, ' output:', labels_test.shape)
# reshaping image tensors to a matrix 
inputs_train = images_train.reshape(-1, 28 * 28)
inputs_test  = images_test.reshape(-1, 28 * 28)

# one-hot-encode labels
outputs_train = OneHotEncoder(sparse=False).fit_transform(labels_train.reshape(-1, 1))
outputs_test  = OneHotEncoder(sparse=False).fit_transform(labels_test.reshape(-1, 1))
print('dimensions:')
print('training input:', inputs_train.shape, ' output:', outputs_train.shape)
print('testing  input:', inputs_test.shape, ' output:', outputs_test.shape)
def test(sess, optimizer, CE, p_train, p_test):
    # training
    error_train = []
    error_test  = []
    for i in range(2000):
        (_,ce,pa,pb) = sess.run([optimizer,CE,p_train,p_test])
        pa = np.argmax(pa,axis=1)
        pa[pa==0] = 4
        pa[pa==1] = 9 
        labels_train_pred = pa
        pb = np.argmax(pb,axis=1)
        pb[pb==0] = 4
        pb[pb==1] = 9    
        labels_test_pred = pb
        err_train = 1 - accuracy_score(labels_train,labels_train_pred)
        err_test  = 1 - accuracy_score(labels_test,labels_test_pred)
        error_train.append(err_train)
        error_test.append(err_test)
        if (i+1) % 100 == 0:
            print('step = %-5d cross_entropy = %-10.5f training = %-10.5f testing = %-10.5f' % 
                  (i+1,ce,err_train,err_test))<jupyter_output><empty_output><jupyter_text>(a) Use logistic regression<jupyter_code># define TensorFlow computation graph

x = tf.constant(inputs_train, dtype='float32', shape=[1000, 784])
x_test = tf.constant(inputs_test, dtype='float32', shape=[1000, 784])
y = tf.constant(outputs_train, dtype='float32', shape=[1000, 2])

w = tf.Variable(tf.truncated_normal([784, 2], stddev=0.1))
b = tf.Variable(tf.truncated_normal([1, 2], stddev=0.1))

logits = tf.matmul(x, w) + b
p_train = tf.nn.softmax(logits)
p_test = tf.nn.softmax(tf.matmul(x_test, w) + b)

CE = tf.reduce_mean(tf.reduce_sum(-y * tf.log(tf.nn.softmax(logits)), reduction_indices=[1]))

optimizer = tf.train.AdamOptimizer().minimize(CE)

# initialization of variables
init = tf.initialize_all_variables()

# initialize a computation session
sess = tf.Session()
sess.run(init)
test(sess, optimizer, CE, p_train, p_test)<jupyter_output>step = 100   cross_entropy = 0.20544    training = 0.04000    testing = 0.05500   
step = 200   cross_entropy = 0.13411    training = 0.02200    testing = 0.04700   
step = 300   cross_entropy = 0.10347    training = 0.01800    testing = 0.04500   
step = 400   cross_entropy = 0.08542    training = 0.01600    testing = 0.04300   
step = 500   cross_entropy = 0.07306    training = 0.01300    testing = 0.04000   
step = 600   cross_entropy = 0.06381    training = 0.01200    testing = 0.03900   
step = 700   cross_entropy = 0.05651    training = 0.01100    testing = 0.03800   
step = 800   cross_entropy = 0.05052    training = 0.01000    testing = 0.03700   
step = 900   cross_entropy = 0.04549    training = 0.00600    testing = 0.03600   
step = 1000  cross_entropy = 0.04120    training = 0.00500    testing = 0.03500   
step = 1100  cross_entropy = 0.03750    training = 0.00500    testing = 0.03500   
step = 1200  cross_entropy = 0.03427    training = 0.00300    testing = 0.03500   
step[...]<jupyter_text>(b) Use a neural network with one hidden layer of 1024 nodes. Do not use a RELU layer between the linear layers.<jupyter_code># define TensorFlow computation graph
epsilon = 1e-15

w1 = tf.Variable(tf.truncated_normal([784, 1024], stddev=0.1))
b1 = tf.Variable(tf.truncated_normal([1, 1024], stddev=0.1))

w2 = tf.Variable(tf.truncated_normal([1024, 2], stddev=0.1))
b2 = tf.Variable(tf.truncated_normal([1, 2], stddev=0.1))

y1_train = tf.matmul(x, w1) + b1
y1_test = tf.matmul(x_test, w1) + b1

y2_train = tf.matmul(y1_train, w2) + b2
y2_test = tf.matmul(y1_test, w2) + b2

p_train = tf.nn.softmax(y2_train) + epsilon
p_test = tf.nn.softmax(y2_test) + epsilon

CE = tf.reduce_mean(tf.reduce_sum(-y * tf.log(p_train), reduction_indices=[1]))

optimizer = tf.train.AdamOptimizer().minimize(CE)

# initialization of variables
init = tf.initialize_all_variables()

# initialize a computation session
sess = tf.Session()
sess.run(init)
# training
test(sess, optimizer, CE, p_train, p_test)<jupyter_output>step = 100   cross_entropy = 0.03213    training = 0.00700    testing = 0.03400   
step = 200   cross_entropy = 0.01645    training = 0.00000    testing = 0.03400   
step = 300   cross_entropy = 0.00944    training = 0.00000    testing = 0.03400   
step = 400   cross_entropy = 0.00577    training = 0.00000    testing = 0.03800   
step = 500   cross_entropy = 0.00371    training = 0.00000    testing = 0.03800   
step = 600   cross_entropy = 0.00250    training = 0.00000    testing = 0.03900   
step = 700   cross_entropy = 0.00175    training = 0.00000    testing = 0.04000   
step = 800   cross_entropy = 0.00127    training = 0.00000    testing = 0.04200   
step = 900   cross_entropy = 0.00096    training = 0.00000    testing = 0.04200   
step = 1000  cross_entropy = 0.00074    training = 0.00000    testing = 0.04200   
step = 1100  cross_entropy = 0.00058    training = 0.00000    testing = 0.04200   
step = 1200  cross_entropy = 0.00047    training = 0.00000    testing = 0.04300   
step[...]<jupyter_text>(c) Use a neural network with one hidden layer of 1024 nodes. Separate the linear layers with a RELU layer.<jupyter_code># define TensorFlow computation graph

w1 = tf.Variable(tf.truncated_normal([784, 1024], stddev=0.1))
b1 = tf.Variable(tf.truncated_normal([1, 1024], stddev=0.1))

w2 = tf.Variable(tf.truncated_normal([1024, 2], stddev=0.1))
b2 = tf.Variable(tf.truncated_normal([1, 2], stddev=0.1))

y1_train = tf.nn.relu(tf.matmul(x, w1) + b1)
y1_test = tf.nn.relu(tf.matmul(x_test, w1) + b1)

y2_train = tf.matmul(y1_train, w2) + b2
y2_test = tf.matmul(y1_test, w2) + b2

p_train = tf.nn.softmax(y2_train)
p_test = tf.nn.softmax(y2_test)

CE = tf.reduce_mean(tf.reduce_sum(-y * tf.log(p_train), reduction_indices=[1]))

optimizer = tf.train.AdamOptimizer().minimize(CE)

# initialization of variables
init = tf.initialize_all_variables()

# initialize a computation session
sess = tf.Session()
sess.run(init)
test(sess, optimizer, CE, p_train, p_test)<jupyter_output>step = 100   cross_entropy = 0.00624    training = 0.00000    testing = 0.02000   
step = 200   cross_entropy = 0.00115    training = 0.00000    testing = 0.02300   
step = 300   cross_entropy = 0.00043    training = 0.00000    testing = 0.02300   
step = 400   cross_entropy = 0.00022    training = 0.00000    testing = 0.02300   
step = 500   cross_entropy = 0.00013    training = 0.00000    testing = 0.02400   
step = 600   cross_entropy = 0.00009    training = 0.00000    testing = 0.02400   
step = 700   cross_entropy = 0.00006    training = 0.00000    testing = 0.02400   
step = 800   cross_entropy = 0.00005    training = 0.00000    testing = 0.02400   
step = 900   cross_entropy = 0.00004    training = 0.00000    testing = 0.02400   
step = 1000  cross_entropy = 0.00003    training = 0.00000    testing = 0.02400   
step = 1100  cross_entropy = 0.00002    training = 0.00000    testing = 0.02400   
step = 1200  cross_entropy = 0.00002    training = 0.00000    testing = 0.02400   
step[...]<jupyter_text>(d) Use a neural network with one hidden layer of 1024 nodes. Separate the linear layers with a RELU layer. Use L2 regularization.<jupyter_code># define TensorFlow computation graph

w1 = tf.Variable(tf.truncated_normal([784, 1024], stddev=0.1))
b1 = tf.Variable(tf.truncated_normal([1, 1024], stddev=0.1))

w2 = tf.Variable(tf.truncated_normal([1024, 2], stddev=0.1))
b2 = tf.Variable(tf.truncated_normal([1, 2], stddev=0.1))

y1_train = tf.nn.relu(tf.matmul(x, w1) + b1)
y1_test = tf.nn.relu(tf.matmul(x_test, w1) + b1)

y2_train = tf.matmul(y1_train, w2) + b2
y2_test = tf.matmul(y1_test, w2) + b2

p_train = tf.nn.softmax(y2_train)
p_test = tf.nn.softmax(y2_test)

beta = 0.00001 # 0.0200
CE = tf.reduce_mean(tf.reduce_sum(-y * tf.log(p_train)
                                  + beta * (tf.reduce_mean(tf.reduce_sum(w1 ** 2))
                                            + tf.reduce_mean(tf.reduce_sum(w2 ** 2))),
                                  reduction_indices=[1]))

optimizer = tf.train.AdamOptimizer().minimize(CE)

# initialization of variables
init = tf.initialize_all_variables()

# initialize a computation session
sess = tf.Session()
sess.run(init)
test(sess, optimizer, CE, p_train, p_test)<jupyter_output>step = 100   cross_entropy = 0.08053    training = 0.00000    testing = 0.02300   
step = 200   cross_entropy = 0.06183    training = 0.00000    testing = 0.02500   
step = 300   cross_entropy = 0.05394    training = 0.00000    testing = 0.02500   
step = 400   cross_entropy = 0.04856    training = 0.00000    testing = 0.02400   
step = 500   cross_entropy = 0.04429    training = 0.00000    testing = 0.02400   
step = 600   cross_entropy = 0.04071    training = 0.00000    testing = 0.02400   
step = 700   cross_entropy = 0.03762    training = 0.00000    testing = 0.02500   
step = 800   cross_entropy = 0.03490    training = 0.00000    testing = 0.02600   
step = 900   cross_entropy = 0.03249    training = 0.00000    testing = 0.02600   
step = 1000  cross_entropy = 0.03032    training = 0.00000    testing = 0.02500   
step = 1100  cross_entropy = 0.02835    training = 0.00000    testing = 0.02600   
step = 1200  cross_entropy = 0.02655    training = 0.00000    testing = 0.02600   
step[...]
{'fraction_non_alphanumeric': 0.12270789969273466, 'fraction_numerical': 0.12746555654673405, 'mean_word_length': 3.310123878684323, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '595083', 'license': 'MIT', 'n_tokens_mistral': 4612, 'n_tokens_neox': 3673, 'n_words': 839, 'path': 'Homework 06.ipynb', 'repo_name': 'derrowap/MA490-Deep-Learning', 'size': '10103'}