<jupyter_start><jupyter_text>Neural NetworkImplement the backpropagation algorithm for neural networks and apply it to the task of hand-written digit recognition.<jupyter_code>import numpy as np
from scipy.io import loadmat
import scipy.optimize as opt
import matplotlib.pyplot as plt
import matplotlib
from sklearn.metrics import classification_report
%matplotlib inline
np.set_printoptions(precision=3)<jupyter_output><empty_output><jupyter_text>1 Load data and parameters<jupyter_code>def load_parameters(path):
    data = loadmat(path)
    return data['Theta1'], data['Theta2']
theta_1, theta_2 = load_parameters("ex4weights.mat")
print(theta_1.shape)
print(theta_2.shape)
def serialize(theta_seq):
    '''
    Serialize theta_1(25, 401) and theta_2(10, 26) to serialized theta(10285,)
    '''
    res = None
    
    for i in range(1, len(theta_seq)):
        res = np.concatenate((theta_seq[0].ravel(), theta_seq[i].ravel()))
    
    return res
theta = serialize([theta_1, theta_2])
theta.shape
def deserialize(theta):
    '''
    Deserialize theta(10285,) to theta_1(25, 401) and theta_2(10, 26)
    '''
    return [theta[:25 * 401].reshape((25, 401)), theta[25 * 401:].reshape((10, 26))]
t1, t2 = deserialize(theta)
print(t1.shape)
print(t2.shape)
def load_data(path):
    data = loadmat(path)
    return data['X'], data['y'].ravel()
def transform_y(y):
    '''
    Transform y from (5000,) to (10, 5000)
    if y_i = 1 then after transformation y_i = [1, 0, 0, 0, 0, 0, 0, 0, 0, 0]
    '''
    y_matrix = []
    for i in range(1, 11):
        y_matrix.append((y == i).astype(int))
    return np.array(y_matrix)
X, y = load_data("ex4data1.mat")
raw_y = y.copy()
print(y.shape)
y = transform_y(y)

print(X.shape)
print(y.shape)
y<jupyter_output>(5000,)
(5000, 400)
(10, 5000)<jupyter_text>2 Feedfoward propagationImplement the foward-propagation algorithm to use trained parameters to predict.<jupyter_code>def sigmoid(z):
    return 1 / (np.exp(-z) + 1)
def foward_propagation(theta, X):
    '''
    Feedfoward propagation algorithm
    '''
    theta = deserialize(theta)
    a = [X]
    z = []
    
    for t in theta:
        a[-1] = np.insert(a[-1], 0, values=np.ones(a[-1].shape[0]), axis=1)
        z.append(a[-1] @ t.T)
        a.append(sigmoid(z[-1]))
        
    return a, z
a, z = foward_propagation(theta, X)

print(a[2].shape)
a[2]<jupyter_output>(5000, 10)<jupyter_text>3 Regularized cost function The cost function of neural network is defined as $$J(\theta)=\frac{1}{m}\sum_{i=1}^{m}\sum_{k=1}^K\left[-y_k^{(i)}\log\left(h_\theta\left(x^{(i)}\right)_k\right)-\left(1-y^{(i)}_k\right)\log\left(1-h_\theta\left(x^{(i)}\right)_k\right)\right]$$where $K = 10$ in this case.<jupyter_code>def cost(theta, X, y):
    '''
    Compute cost function of neural network
    '''
    a, _ = foward_propagation(theta, X)
    error = -np.multiply(y.T, np.log(a[-1])) - np.multiply(1 - y.T, np.log(1 - a[-1]))
    return error.sum() / len(X)
cost(theta, X, y)<jupyter_output><empty_output><jupyter_text>The regularized cost function of neural network is given by $$J(\theta)=\frac{1}{m}\sum_{i=1}^{m}\sum_{k=1}^K\left[-y_k^{(i)}\log\left(h_\theta\left(x^{(i)}\right)_k\right)-\left(1-y^{(i)}_k\right)\log\left(1-h_\theta\left(x^{(i)}\right)_k\right)\right] + \frac{\lambda}{2m}\left[\sum_{j=1}^{25}\sum_{k=1}^{400}\left(\theta_{jk}^{(1)}\right)^2+\sum_{j=1}^{10}\sum_{k=1}^{25}\left(\theta_{jk}^{(2)}\right)^2\right]$$<jupyter_code>def regularized_cost(theta, X, y, reg):
    '''
    Compute regularized cost function of neural network
    '''
    t1, t2 = deserialize(theta)
    reg_t1 = np.power(t1[:, 1:], 2).sum()
    reg_t2 = np.power(t2[:, 1:], 2).sum()
    return cost(theta, X, y) + (reg / (2 * len(X))) * (reg_t1 + reg_t2)
regularized_cost(theta, X, y, reg=1)<jupyter_output><empty_output><jupyter_text>4 Backpropagation<jupyter_code>def sigmoid_gradient(z):
    '''
    Compute sigmoid gradient.
    g'(z) = g(z)(1 - g(z))
    '''
    return sigmoid(z) * (1 - sigmoid(z))
sigmoid_gradient(0)<jupyter_output><empty_output><jupyter_text>Backpropagation algorithm without regularization$$\delta^{(l)} = \begin{cases}h_\theta(x) - y & l = L \\ \left(\theta^{(l)}\right)^T\delta^{(l+1)}\bigodot g'\left(z^{(l)}\right) & l = 2, 3, \dots, L - 1\end{cases} \\ \frac{\partial}{\partial\theta^{(l)}}J(\theta) = D^{(l)}=\frac{1}{m}\Delta^{(l)} = \frac{1}{m}\sum_{i=1}^m\delta^{(l+1)}\left(a^{(l)}\right)^T$$<jupyter_code>def gradient(theta, X, y):
    '''
    Compute gradient without regularization
    '''
    a, z = foward_propagation(theta, X)
    h = a[-1]
    
    theta = deserialize(theta)
    delta = [np.zeros(theta[i].shape) for i in range(len(theta))]    
    
    for i in range(len(X)):
        di = h[i, :] - y.T[i, :]
        ai = a[-2][i, :]
        delta[-1] += di.reshape(len(di), 1) @ ai.reshape(1, len(ai))
        
        for j in range(len(z) - 1, 0, -1):
            zi = np.insert(z[j - 1][i, :], 0, values=np.ones(1))
            di = np.multiply(theta[j].T @ di, sigmoid_gradient(zi))[1:]
            ai = a[j - 1][i, :]
            
            delta[j - 1] += di.reshape(len(di), 1) @ ai.reshape(1, len(ai))
        
    for delta_i in delta:
        delta_i /= len(X)
    
    return serialize(delta)
d1, d2 = deserialize(gradient(theta, X, y))
print(d1.shape, d2.shape)<jupyter_output>(25, 401) (10, 26)<jupyter_text>Backpropagation algorithm with regulariz$$\frac{\partial}{\partial \theta^{(l)}}J(\theta)=D^{(l)}=\begin{cases}\frac{1}{m}\Delta^{(l)} & j = 0 \\ \frac{1}{m}\Delta^{(l)}+\frac{\lambda}{m}\theta^{(l)} & j \geq 1\end{cases}$$<jupyter_code>def regularized_gradient(theta, X, y, reg):
    delta = deserialize(gradient(theta, X, y))
    theta = deserialize(theta)
    
    for i in range(len(delta)):
        t = theta[i]
        t[:, 0] = 0
        term = (reg / len(X)) * t
        delta[i] += term
    
    return serialize(delta)
d1, d2 = deserialize(regularized_gradient(theta, X, y, 1))
print(d1.shape, d2.shape)<jupyter_output>(25, 401) (10, 26)<jupyter_text>5 Neural network training<jupyter_code>def rand_init(size, epsilon):
    '''
    Select values for theta uniformly in [-𝜖, 𝜖]
    '''
    return np.random.uniform(-epsilon, epsilon, size)
def nn(X, y, size):
    theta_0 = rand_init(size, 0.12)
    
    res = opt.minimize(fun=regularized_cost, 
                       x0=theta_0, 
                       args=(X, y, 1), 
                       method='TNC', 
                       jac=regularized_gradient, 
                       options={'maxiter': 400})
    
    return res
res = nn(X, y, len(theta))
res
fin_theta = res.x
def show_accuracy(theta, X, y):
    a, _ = foward_propagation(theta, X)
    h = a[-1]
    y_pred = np.argmax(h, axis=1) + 1
    print(classification_report(y, y_pred))
show_accuracy(fin_theta, X, raw_y)<jupyter_output>precision    recall  f1-score   support

           1       0.97      1.00      0.98       500
           2       0.98      0.96      0.97       500
           3       0.88      0.98      0.93       500
           4       0.98      0.97      0.97       500
           5       1.00      0.73      0.84       500
           6       0.97      0.99      0.98       500
           7       0.89      1.00      0.94       500
           8       0.96      0.98      0.97       500
           9       0.98      0.90      0.94       500
          10       0.92      1.00      0.96       500

   micro avg       0.95      0.95      0.95      5000
   macro avg       0.95      0.95      0.95      5000
weighted avg       0.95      0.95      0.95      5000
{'fraction_non_alphanumeric': 0.17606566057717765, 'fraction_numerical': 0.056261583267143235, 'mean_word_length': 2.7034313725490198, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '124025', 'license': 'MIT', 'n_tokens_mistral': 3107, 'n_tokens_neox': 2839, 'n_words': 591, 'path': 'exercise/ex4_nn/1_nn.ipynb', 'repo_name': 'Zx55/Coursera-AndrewNg-ML', 'size': '7554'}