<jupyter_start><jupyter_text>Define Activate Functions  Sigmoid<jupyter_code>def sigmoid(n):
    return 1 / (1 + np.exp(-n))

xx = np.linspace(start=-9, stop=+9)
yy = sigmoid(xx)

plt.plot(xx, yy)
def sigmoid_derivative(n):
    return sigmoid(n) * (1 - sigmoid(n))

xx = np.linspace(start=-9, stop=+9)
yy = sigmoid_derivative(xx)

plt.plot(xx, yy)<jupyter_output><empty_output><jupyter_text>ReLU<jupyter_code>def relu(n):
    return n * (n > 0)

xx = np.linspace(start=-9, stop=+9)
yy = relu(xx)

plt.plot(xx, yy)
def relu_derivative(n):
    return 1 * (n > 0)

xx = np.linspace(start=-9, stop=+9)
yy = relu_derivative(xx)

plt.plot(xx, yy)
%time tmp = [sigmoid(i) for i in range(1000000)]
%time tmp = [sigmoid_derivative(i) for i in range(1000000)]
%time tmp = [relu(i) for i in range(1000000)]
%time tmp = [relu_derivative(i) for i in range(1000000)]<jupyter_output>CPU times: user 168 ms, sys: 14.6 ms, total: 183 ms
Wall time: 182 ms
CPU times: user 162 ms, sys: 7.57 ms, total: 169 ms
Wall time: 170 ms<jupyter_text>Load Dataset<jupyter_code>from sklearn.datasets import load_digits

mnist = load_digits()
X = mnist["data"]

print(X.shape)
X
y = mnist["target"]

print(y.shape)
y<jupyter_output>(1797,)<jupyter_text>Preprocessing<jupyter_code>from keras.utils import to_categorical

y_hot = to_categorical(y)

print(y_hot.shape)
y_hot<jupyter_output>Using TensorFlow backend.<jupyter_text>Train<jupyter_code>import time

def train(X, y, y_hot, num_epoch, learning_rate, activation, activation_derivative):
    num_input = X.shape[1]
    num_output = y_hot.shape[1]

    w1 = np.random.uniform(low=-np.sqrt(6 / (num_input + 100)), high=np.sqrt(6 / (num_input + 100)), size=(num_input, 100))
    w2 = np.random.uniform(low=-np.sqrt(6 / (100 + 100)), high=np.sqrt(6 / (100 + 100)), size=(100, 100))
    w3 = np.random.uniform(low=-np.sqrt(6 / (100 + num_output)), high=np.sqrt(6 / (100 + 10)), size=(100, num_output))

    history = []

    start = time.time()

    for epoch in range(num_epoch):
        z1 = X.dot(w1)
        a1 = activation(z1)
        z2 = a1.dot(w2)
        a2 = activation(z2)
        z3 = a2.dot(w3)
        a3 = sigmoid(z3)

        y_predict = np.argmax(a3, axis=1)    
        error = (y != y_predict).mean()

        if epoch % 10 == 0:
            print("{0:2} error = {1:.5f}".format(epoch, error))

        end = time.time()
        history.append({
            'time': end - start,
            'error': error
        })

        d3 = a3 - y_hot
        d2 = d3.dot(w3.T) * activation_derivative(a2)
        d1 = d2.dot(w2.T) * activation_derivative(a1)

        w3 = w3 - learning_rate * a2.T.dot(d3)
        w2 = w2 - learning_rate * a1.T.dot(d2)
        w1 = w1 - learning_rate * X.T.dot(d1)

    end = time.time()
    elapsed = end - start

    return history, error, elapsed<jupyter_output><empty_output><jupyter_text>Sigmoid<jupyter_code>num_epoch = 100
learning_rate = 0.0003

sigmoid_history, error, elapsed = train(X, y, y_hot, num_epoch, learning_rate,
                                        activation=sigmoid,
                                        activation_derivative=sigmoid_derivative)

print("----" * 10)
print("[sigmoid] Elapsed {0:.2f} seconds, error = {1:.5f}".format(elapsed, error))<jupyter_output>0 error = 0.90317
10 error = 0.35114
20 error = 0.38119
30 error = 0.40790
40 error = 0.38564
50 error = 0.35615
60 error = 0.35225
70 error = 0.29772
80 error = 0.26377
90 error = 0.23150
----------------------------------------
[sigmoid] Elapsed 1.55 seconds, error = 0.19533<jupyter_text>ReLU<jupyter_code>num_epoch = 100
learning_rate = 0.00001

relu_history, error, elapsed = train(X, y, y_hot, num_epoch, learning_rate,
                                     activation=relu,
                                     activation_derivative=relu_derivative)

print("----" * 10)
print("[ReLU] Elapsed {0:.2f} seconds, error = {1:.5f}".format(elapsed, error))
%matplotlib inline
import pandas as pd

sigmoid_history = pd.DataFrame(sigmoid_history)
relu_history = pd.DataFrame(relu_history)

plt.plot(sigmoid_history["time"], sigmoid_history["error"], label="sigmoid")
plt.plot(relu_history["time"], relu_history["error"], label="relu")
plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15531914893617021, 'fraction_numerical': 0.06619385342789598, 'mean_word_length': 3.152109911678116, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '570225', 'license': 'MIT', 'n_tokens_mistral': 1838, 'n_tokens_neox': 1646, 'n_words': 341, 'path': 'example/01-Activation-Functions.ipynb', 'repo_name': 'daesookimds/Deep-Learning', 'size': '4230'}