<jupyter_start><jupyter_text>수치 미분을 이용한 심층 신경망 학습<jupyter_code>import time
import numpy as np<jupyter_output><empty_output><jupyter_text>유틸리티 함수<jupyter_code>epsilon = 0.0001

def _t(x):
    return np.transpose(x)

def _m(A, B):
    return np.matmul(A, B)

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def mean_squared_error(h, y):
    return 1 / 2 * np.mean(np.square(h - y))<jupyter_output><empty_output><jupyter_text>뉴런 구현<jupyter_code>class Neuron:
    def __init__(self, W, b, a):
        # Model Parameter
        self.W = W
        self.b = b
        self.a = a
        
        # Gradients
        self.dW = np.zeros_like(self.W)
        self.db = np.zeros_like(self.b)

    def __call__(self, x):
        return self.a(_m(_t(self.W), x) + self.b) # activation((W^T)x + b)<jupyter_output><empty_output><jupyter_text>심층신경망 구현<jupyter_code>class DNN:
    def __init__(self, hidden_depth, num_neuron, num_input, num_output, activation=sigmoid):
        def init_var(i, o):
            return np.random.normal(0.0, 0.01, (i, o)), np.zeros((o,))

        self.sequence = list()
        # First hidden layer
        W, b = init_var(num_input, num_neuron)
        self.sequence.append(Neuron(W, b, activation))
        
        # Hidden layers
        for _ in range(hidden_depth - 1):
            W, b = init_var(num_neuron, num_neuron)
            self.sequence.append(Neuron(W, b, activation))

        # Output layer
        W, b = init_var(num_neuron, num_output)
        self.sequence.append(Neuron(W, b, activation))

    def __call__(self, x):
        for layer in self.sequence:
            x = layer(x)
        return x

    def calc_gradient(self, x, y, loss_func):
        def get_new_sequence(layer_index, new_neuron):
            new_sequence = list()
            for i, layer in enumerate(self.sequence):
                if i == layer_index:
                    new_sequence.append(new_neuron)
                else:
                    new_sequence.append(layer)
            return new_sequence
        
        def eval_sequence(x, sequence):
            for layer in sequence:
                x = layer(x)
            return x
        
        loss = loss_func(self(x), y) # self(x) == self.__call__(x)
        
        for layer_id, layer in enumerate(self.sequence): # iterate layer
            for w_i, w in enumerate(layer.W): # iterate W (row)
                for w_j, ww in enumerate(w): # iterate W (col)
                    W = np.copy(layer.W)
                    W[w_i][w_j] = ww + epsilon
                    
                    new_neuron = Neuron(W, layer.b, layer.a)
                    new_seq = get_new_sequence(layer_id, new_neuron)
                    h = eval_sequence(x, new_seq)
                    
                    num_grad = (loss_func(h, y) - loss) / epsilon  # (f(x+eps) - f(x)) / epsilon
                    layer.dW[w_i][w_j] = num_grad
            
                for b_i, bb in enumerate(layer.b): # iterate b
                    b = np.copy(layer.b)
                    b[b_i] = bb + epsilon
                    
                    new_neuron = Neuron(layer.W, b, layer.a)
                    new_seq = get_new_sequence(layer_id, new_neuron)
                    h = eval_sequence(x, new_seq)
                    
                    num_grad = (loss_func(h, y) - loss) / epsilon  # (f(x+eps) - f(x)) / epsilon
                    layer.db[b_i] = num_grad
        return loss<jupyter_output><empty_output><jupyter_text>경사하강 학습법<jupyter_code>def gradient_descent(network, x, y, loss_obj, alpha=0.01):
    loss = network.calc_gradient(x, y, loss_obj)
    for layer in network.sequence:
        layer.W += -alpha * layer.dW
        layer.b += -alpha * layer.db
    return loss<jupyter_output><empty_output><jupyter_text>동작 테스트<jupyter_code>x = np.random.normal(0.0, 1.0, (10,))
y = np.random.normal(0.0, 1.0, (2,))

dnn = DNN(hidden_depth=5, num_neuron=32, num_input=10, num_output=2, activation=sigmoid)

t = time.time()
for epoch in range(100):
    loss = gradient_descent(dnn, x, y, mean_squared_error, 0.01)
    print('Epoch {}: Test loss {}'.format(epoch, loss))
print('{} seconds elapsed.'.format(time.time() - t))<jupyter_output>Epoch 0: Test loss 0.5788407666374237
Epoch 1: Test loss 0.5755734143209582
Epoch 2: Test loss 0.5723271009377402
Epoch 3: Test loss 0.5691025121018389
Epoch 4: Test loss 0.565900303856321
Epoch 5: Test loss 0.5627211022628145
Epoch 6: Test loss 0.5595655030828444
Epoch 7: Test loss 0.5564340715483467
Epoch 8: Test loss 0.5533273422196228
Epoch 9: Test loss 0.5502458189274984
Epoch 10: Test loss 0.5471899747975388
Epoch 11: Test loss 0.5441602523521142
Epoch 12: Test loss 0.5411570636878636
Epoch 13: Test loss 0.5381807907236237
Epoch 14: Test loss 0.5352317855158888
Epoch 15: Test loss 0.5323103706372936
Epoch 16: Test loss 0.5294168396136355
Epoch 17: Test loss 0.5265514574157897
Epoch 18: Test loss 0.5237144610016138
Epoch 19: Test loss 0.5209060599039174
Epoch 20: Test loss 0.5181264368597148
Epoch 21: Test loss 0.5153757484771767
Epoch 22: Test loss 0.5126541259349413
Epoch 23: Test loss 0.5099616757111709
Epoch 24: Test loss 0.5072984803367891
Epoch 25: Test loss 0.50466459916995[...]
{'fraction_non_alphanumeric': 0.10873749037721324, 'fraction_numerical': 0.10046189376443418, 'mean_word_length': 2.3080840229153408, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '292589', 'license': 'MIT', 'n_tokens_mistral': 2278, 'n_tokens_neox': 1929, 'n_words': 436, 'path': 'RNN_TF20/04. Deep Neural Network Training with Numerical Gradient-Antonio.ipynb', 'repo_name': '46gom/DLND_Exercises', 'size': '5196'}