<jupyter_start><jupyter_text>Machine Learning - Assignment 9 - Neural Network<jupyter_code>import pandas as pd
import numpy as np
from random import randint
from math import exp
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
def show_numbers(X):
    num_samples = 90
    indices = np.random.choice(range(len(X)), num_samples)
    sample_digits = X[indices]

    fig = plt.figure(figsize=(20, 6))

    for i in range(num_samples):
        ax = plt.subplot(6, 15, i + 1)
        img = 1-sample_digits[i].reshape((16, 16)) #(1-sample_digits[i].reshape((16, 16)))/2*255
        plt.imshow(img, cmap='gray')
        plt.axis('off')<jupyter_output><empty_output><jupyter_text>(a) Optimize width (the number of neurons in a hidden layer; it is usually the same for all of them) and depth of the network. Try to find a setting that trains in a reasonable time. Plot the loss.<jupyter_code>'''
sigmoid function and its derivative
'''
def sigmoid(x):
    return (1 / (1 + np.exp(-x)))

def sigmoid_derivative(x):
    return (sigmoid(x) * (1 - sigmoid(x)))
class NeuralNetwork():
    def __init__(self, X_train, y_train, size):
        size.insert(0,X_train.shape[1])
        self.size = size
        self.layers = len(self.size)
        self.X_train, self.y_train = X_train, y_train
 
        self.weights = []
        
        for i in range(self.layers-1):
            self.weights.append(np.random.randn(self.size[i],self.size[i+1]))
        self.success_rates = []
            
    '''
    calculate f(x) and f'(x) and store them temporily
    '''
    def feed_forward(self, data):
        sigmoid_v = sigmoid
        sigmoid_d_v = sigmoid_derivative
        
        self.sigmoid_results = [data]
        self.sigmoid_d_results = []
        
        for i in range(self.layers-1):
            self.sigmoid_results.append([])
            self.sigmoid_d_results.append([])
        
        for layer in range(self.layers - 1):
            for row in range(data.shape[0]):
                product = np.dot(self.sigmoid_results[layer][row], self.weights[layer])
                self.sigmoid_results[layer + 1].append(sigmoid_v(product))
                self.sigmoid_d_results[layer].append(sigmoid_d_v(product))
                
    def error_derivative(self):
        self.errors = np.array(self.sigmoid_results[-1]) - self.y_train
        
    def backprop(self):
        self.deltas = []
        
        for layer in range(self.layers - 1):
            self.deltas.append([])        
        
        for row in range(self.X_train.shape[0]):
            d = np.diag(self.sigmoid_d_results[-1][row])
            err = self.errors[row].reshape(self.errors.shape[1], 1)
            delta = np.dot(d,err)
            self.deltas[0].append(delta)
            
            for x in range(self.layers-2):
                d_i = -(x+2)
                w_i = -(x+1)
                d = np.diag(self.sigmoid_d_results[d_i][row])
                w = self.weights[w_i]
                delta_i = self.deltas[x][row]
                delta = np.dot(np.dot(d,w), delta_i)
                self.deltas[x+1].append(delta)
            
    def correct(self, gamma):
        for row in range(self.X_train.shape[0]):
            for x in range(self.layers-1):
                d = self.deltas[x][row].T
                o_i = -(x+2)
                wt_i = -(x+1)
                o = self.sigmoid_results[o_i][row].reshape(self.sigmoid_results[o_i][row].shape[0], 1)
                Delta = -gamma * np.dot(o,d)
                self.weights[wt_i] = self.weights[wt_i] + Delta
            
    def check_class(self):
        self.class_check = np.zeros((self.y_train.shape))
        for row in range(self.y_train.shape[0]):
            self.class_check[row,int(self.sigmoid_results[-1][row].argmax())] += 1
            
    def accuracy_check(self):
        total = self.y_train.shape[0]
        correct = 0
        for row in range(total):
            if np.array_equal(self.y_train[row], self.class_check[row]):
                correct += 1
        return (correct / total)
    
    def train(self,iterations=100, gamma=0.05, printfreq=25):
        for iteration in range(iterations):
            self.feed_forward(self.X_train)
            self.check_class()
            if iteration % printfreq == 0 or iteration == iterations - 1:
                print("Iteration {} | Accuracy: {}".format(iteration,self.accuracy_check()))
            self.success_rates.append(self.accuracy_check())
            self.error_derivative()
            self.backprop()
            self.correct(gamma)
            
    def predict(self,X_test,y_test):
        self.X_test, self.y_test = X_test, y_test
        self.feed_forward(self.X_test)
        predictions = []
        
        for row in range(self.X_test.shape[0]):
            predictions.append(int(self.sigmoid_results[-1][row].argmax()))
        self.pred_array = np.array(predictions)
        
    def error_rate(self):
        bools = np.equal(self.pred_array, self.y_test.flatten())
        correct = np.sum(bools)
        return (self.y_test.shape[0] - correct) / self.y_test.shape[0]
trainset = np.loadtxt('./Data/zip.train')

constant_coefficient = np.ones((trainset.shape[0],1))
X_train = np.concatenate((trainset[:,1:], constant_coefficient),axis=-1)
y_train = np.zeros((trainset.shape[0],10))
for i in range(trainset.shape[0]):
    y_train[i,int(trainset[i,0])] += 1<jupyter_output><empty_output><jupyter_text>Comments: both processes of training model take pretty much time... Especially the second one takes me at home ~36min to finish. You can check the result in pdf :)<jupyter_code>nn = NeuralNetwork(X_train, y_train, [10,10])
nn.train(gamma = 0.0008)
nn2 = NeuralNetwork(X_train, y_train, [100,50,10])
nn2.train(iterations = 500,gamma = 0.001)
testset = np.loadtxt('./Data/zip.test')
constant_coefficient = np.ones((testset.shape[0],1))
X_test = np.concatenate((testset[:,1:],constant_coefficient),axis=-1)
y_test = testset[:,0]
X_test.shape
y_test.shape
x = list(range(1,501))
y = list(map((lambda x: x*100), nn2.success_rates))
plt.xlabel("Iteration Number")
plt.ylabel("Accuracy")
plt.plot(x, y)<jupyter_output><empty_output><jupyter_text>(b) Show some digits that are classified incorrectly<jupyter_code>np.mean(nn2.pred_array==y_test)
indices = np.where(nn2.pred_array != test_labels)
show_numbers(X_test[indices])<jupyter_output><empty_output><jupyter_text>(c) Plot your first weight layer as a grayscale image<jupyter_code>first_layer = np.array(nn2.weights[0][:,1:])
plt.imshow(first_layer, cmap='gray')
plt.axis('off')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13566243194192376, 'fraction_numerical': 0.019207501512401692, 'mean_word_length': 2.6018518518518516, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '279663', 'license': 'Apache-2.0', 'n_tokens_mistral': 2358, 'n_tokens_neox': 2248, 'n_words': 427, 'path': 'Machine Learning/U9/Neuralnetwork.ipynb', 'repo_name': 'qiaw99/WS2020-21', 'size': '6612'}