<jupyter_start><jupyter_text>What to dochapter 4 of the fish book  Loss functions  Mean squared error<jupyter_code>import numpy as np
def mean_squared_error(y,t):
    return 0.5*np.sum((y-t)**2)
t = [0, 0, 1, 0, 0, 0, 0, 0, 0, 0]
y1 = [0.1, 0.05, 0.6, 0.0, 0.0, 0.1, 0.0, 0.1, 0.0, 0.0]
y2 = [0.1, 0.05, 0.1, 0.0, 0.05, 0.1, 0.0, 0.6, 0.0, 0.0]
mean_squared_error(np.array(y1), np.array(t))
mean_squared_error(np.array(y2), np.array(t))<jupyter_output><empty_output><jupyter_text>Cross Entropy Error<jupyter_code>def cross_entropy_error(y,t):
    delta = 1e-7
    return -np.sum(t*np.log(y + delta))
cross_entropy_error(np.array(y1), np.array(t))
cross_entropy_error(np.array(y2), np.array(t))<jupyter_output><empty_output><jupyter_text>Mini batch<jupyter_code>import sys, os
sys.path.append(os.pardir)
from dataset.mnist import load_mnist
(x_train, t_train), (x_test, y_test) = load_mnist(normalize=True, one_hot_label=True)
x_train.shape
t_train.shape
train_size = x_train.shape[0]
batch_size = 10
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
np.random.choice(60000, 10)<jupyter_output><empty_output><jupyter_text>Mini batch cross entropy - one hot<jupyter_code>def cross_entropy_error(y, t):
    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)
       
    batch_size = y.shape[0]
    return -np.sum(t*np.log(y+1e-7))/batch_size<jupyter_output><empty_output><jupyter_text>- in the case of labels<jupyter_code>def cross_entropy_error(y, t):
    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)
       
    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7))/batch_size<jupyter_output><empty_output><jupyter_text>Numerical Differentiation<jupyter_code># Bad one 
def numerical_diff(f, x):
    h = 10e-50 # too small
    return(f(x+h)-f(x))/h #one sided
def x_3(x):
    return x**3
numerical_diff(x_3, 1)
# Better
def numerical_diff(f, x):
    h = 1e-4 #0.0001
    return (f(x+h)-f(x-h))/(2*h)
numerical_diff(x_3, 1)
def function_1(x):
    return 0.01*x**2 + 0.1*x
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
x = np.arange(0.0, 20.0, 0.1)
y = function_1(x)
plt.plot(x,y)
numerical_diff(function_1, 5)
numerical_diff(function_1, 10)
def tangent_line(f, x):
    d = numerical_diff(f, x)
    print(d)
    y = f(x) - d*x
    return lambda t: d*t + y
tf = tangent_line(function_1, 5)
y2 = tf(x)
plt.plot(x,y)
plt.plot(x, y2)<jupyter_output><empty_output><jupyter_text>Partial Differntial<jupyter_code>def function_2(x):
    return x[0]**2 + x[1]**2
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
import numpy as np
fig = plt.figure()
ax = fig.gca(projection = '3d')
x0 = np.arange(-3,3,0.1)
X, Y = np.meshgrid(x0, x0)
surf = ax.plot_surface(X, Y, function_2([X,Y]), linewidth = 0, cmap = 'bwr')
fig.colorbar(surf)
fig
def function_tmp1(x0):
    return x0*x0 + 4.0**2.0
numerical_diff(function_tmp1, 3.0)
def function_tmp2(x1):
    return x1*x1 + 3.0**2.0
numerical_diff(function_tmp2, 4.0)<jupyter_output><empty_output><jupyter_text>gradients<jupyter_code>def numerical_gradient(f, x):
    h = 1e-4
    grad = np.zeros_like(x)
    
    for idx in range(x.size):
        tmp_val = x[idx]
        # f(x+h)の計算
        x[idx] = tmp_val + h
        fxh1 = f(x)
        # f(x-h)
        x[idx] = tmp_val - h
        fxh2 = f(x)
        
        grad[idx] = (fxh1-fxh2)/(2*h)
        x[idx] = tmp_val
       
    return grad
numerical_gradient(function_2, np.array([3.0, 4.0]))
numerical_gradient(function_2, np.array([0.0, 2.0]))
numerical_gradient(function_2, np.array([3.0, 0.0]))
def _numerical_gradient_no_batch(f, x):
    h = 1e-4 # 0.0001
    grad = np.zeros_like(x)
    
    for idx in range(x.size):
        tmp_val = x[idx]
        x[idx] = float(tmp_val) + h
        fxh1 = f(x) # f(x+h)
        
        x[idx] = tmp_val - h 
        fxh2 = f(x) # f(x-h)
        grad[idx] = (fxh1 - fxh2) / (2*h)
        
        x[idx] = tmp_val # 値を元に戻す
        
    return grad
def numerical_gradient(f, X):
    if X.ndim == 1:
        return _numerical_gradient_no_batch(f, X)
    else:
        grad = np.zeros_like(X)
        
        for idx, x in enumerate(X):
            grad[idx] = _numerical_gradient_no_batch(f, x)
        
        return grad
def function_2(x):
    if x.ndim == 1:
        return np.sum(x**2)
    else:
        return np.sum(x**2, axis=1)
def tangent_line(f, x):
    d = numerical_gradient(f, x)
    print(d)
    y = f(x) - d*x
    return lambda t: d*t + y

x0 = np.arange(-2, 2.5, 0.25)
x1 = np.arange(-2, 2.5, 0.25)
X, Y = np.meshgrid(x0, x1)

X = X.flatten()
Y = Y.flatten()

grad = numerical_gradient(function_2, np.array([X, Y]) )

plt.figure()
plt.quiver(X, Y, -grad[0], -grad[1],  angles="xy",color="#666666")#,headwidth=10,scale=40,color="#444444")
plt.xlim([-2, 2])
plt.ylim([-2, 2])
plt.xlabel('x0')
plt.ylabel('x1')
plt.grid()<jupyter_output><empty_output><jupyter_text>Gradient descent<jupyter_code>def gradient_descent(f, init_x, lr = 0.01, step_num = 100):
    x = init_x
    
    for i in range(step_num):
        grad = numerical_gradient(f, x)
        x -= lr *grad
    
    return x
init_x = np.array([-3.0, 4.0])
gradient_descent(function_2, init_x = init_x, lr = 0.1, step_num = 100)

gradient_descent(function_2, init_x = init_x, lr = 10, step_num = 100)
gradient_descent(function_2, init_x = init_x, lr = 1e-8, step_num = 100)<jupyter_output><empty_output><jupyter_text>Gradients in neural networks  Simple nets<jupyter_code>%run functions.py
%run gradient.py
class simpleNet:
    def __init__(self):
        self.W = np.random.randn(2,3)
        
    def predict(self, x):
        return np.dot(x, self.W)
    
    def loss(self, x, t):
        z = self.predict(x)
        y = softmax(z)
        loss = cross_entropy_error(y, t)
        
        return loss
net = simpleNet()
net.W
x = np.array([0.6, 0.9])
p = net.predict(x)
p
np.argmax(p)
t = np.array([0, 0, 1])
net.loss(x, t)<jupyter_output><empty_output><jupyter_text>gradients<jupyter_code>def f(W):
    return net.loss(x, t)
dW = numerical_gradient(f, net.W)
dW<jupyter_output><empty_output><jupyter_text>Learning Algorithms  stochastic gradient descent<jupyter_code>class TwoLayerNet:
    def __init__(self, input_size, hidden_size, output_size, weight_init_std = 0.01):
        # initialize weights
        self.params = {}
        self.params['W1'] = weight_init_std * np.random.randn(input_size, hidden_size)
        self.params['b1'] = np.zeros(hidden_size)
        self.params['W2'] = weight_init_std * np.random.randn(hidden_size, output_size)
        self.params['b2'] = np.zeros(output_size)
    
    def predict(self, x):
        W1, W2 = self.params['W1'], self.params['W2']
        b1, b2 = self.params['b1'], self.params['b2']
        
        a1 = np.dot(x, W1) + b1
        z1 = sigmoid(a1)
        a2 = np.dot(z1, W2) + b2
        y = softmax(a2)
        
        return y
    
    def loss(self, x, t):
        y = self.predict(x)
        
        return cross_entropy_error(y, t)
    
    def accuracy(self, x, t):
        y = self.predict(x)
        y = np.argmax(y, axis = 1)
        t = np.argmax(t, axis = 1)
        
        accuracy = np.sum(y == t)/float(x.shape[0])
        return accuracy
    
    def numerical_gradient(self, x, t):
        loss_W = lambda W: self.loss(x,t)
        
        grads = {}
        grads['W1'] = numerical_gradient(loss_W, self.params['W1'])
        grads['b1'] = numerical_gradient(loss_W, self.params['b1'])
        grads['W2'] = numerical_gradient(loss_W, self.params['W2'])
        grads['b2'] = numerical_gradient(loss_W, self.params['b2'])
        
        return grads
    
    def gradient(self, x, t):
        W1, W2 = self.params['W1'], self.params['W2']
        b1, b2 = self.params['b1'], self.params['b2']
        grads = {}
        
        batch_num = x.shape[0]
        
        # forward
        a1 = np.dot(x, W1) + b1
        z1 = sigmoid(a1)
        a2 = np.dot(z1, W2) + b2
        y = softmax(a2)
        
        # backward
        dy = (y-t)/batch_num
        grads['W2'] = np.dot(z1.T, dy)
        grads['b2'] = np.sum(dy, axis = 0)
        
        da1 = np.dot(dy, W2.T)
        dz1 = sigmoid_grad(a1)*da1
        grads['W1'] = np.dot(x.T, dz1)
        grads['b1'] = np.sum(dz1, axis = 0)
        
        return grads
net = TwoLayerNet(input_size = 784, hidden_size=100, output_size=10)

print(net.params['W1'].shape)
print(net.params['b1'].shape)
print(net.params['W2'].shape)
print(net.params['b2'].shape)
x = np.random.rand(100, 784)
y = net.predict(x)
x = np.random.rand(100, 784)
t = np.random.rand(100, 10)

grads = net.numerical_gradient(x, t)
print(grads['W1'].shape)
print(grads['b1'].shape)
print(grads['W2'].shape)
print(grads['b2'].shape)
1<jupyter_output><empty_output><jupyter_text>Mini batch (SGD)<jupyter_code>(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, one_hot_label=True)
train_loss_list = []


iters_num = 10000
train_size = x_train.shape[0]
batch_size = 100

learning_rate = 0.1
network = TwoLayerNet(input_size = 784, hidden_size=50, output_size=10)

import time
for i in range(iters_num):
    # time test
    if i == 0:
        ts = time.time()
    if i == 1:
        secs = time.time() - ts
        print("One run took " + str(secs) + " seconds, " + str(iters_num) + " runs will take an estimated " + str(secs*iters_num) + " seconds.")
        
    # mini batch
    batch_mask = np.random.choice(train_size, batch_size)
    x_batch = x_train[batch_mask]
    t_batch = t_train[batch_mask]
    
    # Gradient
    #grad = network.numerical_gradient(x_batch, t_batch)
    grad = network.gradient(x_batch, t_batch) # faster
    
    # Update parameters
    for key in ('W1', 'b1', 'W2', 'b2'):
        network.params[key] -= learning_rate *grad[key]
        
    # Learning process
    loss = network.loss(x_batch, t_batch)
    train_loss_list.append(loss)
        
secs = time.time() - ts
print("Took " +str(secs) + " in total.")
fig = plt.figure()
plt.plot(range(0, iters_num),train_loss_list)
plt.xlabel("Iterations")
plt.ylabel("Loss")
plt.xlim(0, 10000)
plt.title("Learning rates")
np.exp(-0.5)<jupyter_output><empty_output><jupyter_text>Check on test data<jupyter_code>(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, one_hot_label=True)

train_loss_list = []
train_acc_list = []
test_acc_list = []
iter_per_epoch = max(train_size/batch_size, 1)
iters_num = 10000
batch_size = 100
learning_rate = 0.1

network = TwoLayerNet(input_size = 784, hidden_size=50, output_size=10)
for i in range(iters_num):
    # mini batch 
    batch_mask = np.random.choice(train_size, batch_size)
    x_batch = x_train[batch_mask]
    t_batch = t_train[batch_mask]
    
    # Gradient
    #grad = network.numerical_gradient(x_batch, t_batch)
    grad = network.gradient(x_batch, t_batch)
    
    # Parameters
    for key in ('W1', "b1", "W2", "b2"):
        network.params[key] -= learning_rate * grad[key]
        
    loss = network.loss(x_batch, t_batch)
    train_loss_list.append(loss)
    
    if i % iter_per_epoch == 0:
        train_acc = network.accuracy(x_train, t_train)
        test_acc = network.accuracy(x_test, t_test)
        train_acc_list.append(train_acc)
        test_acc_list.append(test_acc)
        print("train acc, test acc| " + str(train_acc) + ", " + str(test_acc))
        
        
        
        
len(train_acc_list)
fig = plt.figure()
plt.plot(range(len(train_acc_list)), train_acc_list, label = "train")
plt.plot(range(len(train_acc_list)), test_acc_list, linestyle = "--", label = "test")
plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15209286561599727, 'fraction_numerical': 0.037705473648534146, 'mean_word_length': 3.105391304347826, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '184715', 'license': 'FTL', 'n_tokens_mistral': 5084, 'n_tokens_neox': 4831, 'n_words': 917, 'path': 'src/code/Chapter4.ipynb', 'repo_name': 'banbiossa/deep-learning-from-scrach', 'size': '11802'}