<jupyter_start><jupyter_text>Load Dataset<jupyter_code>from sklearn.datasets import make_blobs

X, y = make_blobs(n_samples=1000,
                  n_features=64,
                  centers=10,
                  center_box=(-1, +1))

print(X.shape, y.shape)
X[0]
y[0:5]<jupyter_output><empty_output><jupyter_text>Visualize<jupyter_code># plt.scatter(X[:, 0], X[:, 1], c=y.ravel())
X.mean(), X.min(), X.max()

plt.hist(X.ravel(), bins=100)<jupyter_output><empty_output><jupyter_text>Preprocessing  One Hot Encode Label<jupyter_code>y_hot = np.eye(10)[y]

print(y.shape, y_hot.shape)
y_hot[:10]<jupyter_output>(1000,) (1000, 10)<jupyter_text>Define Activate Functions  Sigmoid<jupyter_code>def sigmoid(n):
    return 1 / (1 + np.exp(-n))

xx = np.linspace(-9, +9)
yy = sigmoid(xx)

plt.plot(xx, yy)
def sigmoid_derivative(n):
    return sigmoid(n) * (1 - sigmoid(n))

xx = np.linspace(-9, +9)
yy = sigmoid_derivative(xx)

plt.plot(xx, yy)<jupyter_output><empty_output><jupyter_text>tanh<jupyter_code>def tanh(x):
    return np.tanh(x)

xx = np.linspace(-9, +9)
yy = tanh(xx)

plt.plot(xx, yy)
def tanh_derivative(x):
    return 1.0 - np.tanh(x) ** 2

xx = np.linspace(-9, +9)
yy = tanh_derivative(xx)

plt.plot(xx, yy)<jupyter_output><empty_output><jupyter_text>ReLU<jupyter_code>def relu(n):
    return n * (n > 0)

xx = np.linspace(start=-9, stop=+9)
yy = relu(xx)

plt.plot(xx, yy)
def relu_derivative(n):
    return 1 * (n > 0)

xx = np.linspace(start=-9, stop=+9)
yy = relu_derivative(xx)

plt.plot(xx, yy)<jupyter_output><empty_output><jupyter_text>Plot<jupyter_code>def plot_w(ws, range_):
    figure, axes = plt.subplots(nrows=1, ncols=5)
    figure.set_size_inches(24, 4)

    axes[0].hist(ws[0].ravel(), bins=100, range=range_)
    axes[1].hist(ws[1].ravel(), bins=100, range=range_)
    axes[2].hist(ws[2].ravel(), bins=100, range=range_)
    axes[3].hist(ws[3].ravel(), bins=100, range=range_)
    axes[4].hist(ws[4].ravel(), bins=100, range=range_)
def plot_a(as_, range_=None):
    figure, axes = plt.subplots(nrows=1, ncols=5)
    figure.set_size_inches(24, 4)

    axes[0].hist(as_[0].ravel(), bins=100, range=range_)
    axes[1].hist(as_[1].ravel(), bins=100, range=range_)
    axes[2].hist(as_[2].ravel(), bins=100, range=range_)
    axes[3].hist(as_[3].ravel(), bins=100, range=range_)
    axes[4].hist(as_[4].ravel(), bins=100, range=range_)
def plot_dzs(dzs, range_=None):
    figure, axes = plt.subplots(nrows=1, ncols=5)
    figure.set_size_inches(24, 4)

    axes[0].hist(dzs[0].ravel(), bins=100, range=range_)
    axes[1].hist(dzs[1].ravel(), bins=100, range=range_)
    axes[2].hist(dzs[2].ravel(), bins=100, range=range_)
    axes[3].hist(dzs[3].ravel(), bins=100, range=range_)
def plot_ds(ds, range_=None, exclude=0.0):
    figure, axes = plt.subplots(nrows=1, ncols=5)
    figure.set_size_inches(24, 4)

    axes[0].hist(ds[4][(ds[4] > exclude) | (ds[4] < -exclude)], bins=100, range=range_)
    axes[1].hist(ds[3][(ds[3] > exclude) | (ds[3] < -exclude)], bins=100, range=range_)
    axes[2].hist(ds[2][(ds[2] > exclude) | (ds[2] < -exclude)], bins=100, range=range_)
    axes[3].hist(ds[1][(ds[1] > exclude) | (ds[1] < -exclude)], bins=100, range=range_)
    axes[4].hist(ds[0][(ds[0] > exclude) | (ds[0] < -exclude)], bins=100, range=range_)<jupyter_output><empty_output><jupyter_text>Build a Model<jupyter_code>def train(X, y, num_epoch, learning_rate, ws, bs, activation, activation_derivative):
    w1, w2, w3, w4, w5 = ws[0], ws[1], ws[2], ws[3], ws[4]
    b1, b2, b3, b4, b5 = bs[0], bs[1], bs[2], bs[3], bs[4]

    history = []
    
    for epoch in range(num_epoch):
        z1 = X.dot(w1) + b1
        a1 = activation(z1)
        z2 = a1.dot(w2) + b2
        a2 = activation(z2)
        z3 = a2.dot(w3) + b3
        a3 = activation(z3)
        z4 = a3.dot(w4) + b4
        a4 = activation(z4)
        z5 = a4.dot(w5) + b5
        a5 = sigmoid(z5)

        y_predict = np.argmax(a5, axis=1)
        error = (y_predict != y).mean()

        history.append({
            'epoch': epoch,
            'error': error
        })

        if epoch % (num_epoch / 10) == 0:
            print("{0:3} error = {1:.5f}".format(epoch, error))

        if error < 0.01:
            break

        d5 = (a5 - y_hot)

        d4 = d5.dot(w5.T) * activation_derivative(a4)
        d3 = d4.dot(w4.T) * activation_derivative(a3)
        d2 = d3.dot(w3.T) * activation_derivative(a2)
        d1 = d2.dot(w2.T) * activation_derivative(a1)

        w5 = w5 - learning_rate * a4.T.dot(d5)
        w4 = w4 - learning_rate * a3.T.dot(d4)
        w3 = w3 - learning_rate * a2.T.dot(d3) 
        w2 = w2 - learning_rate * a1.T.dot(d2)
        w1 = w1 - learning_rate * X.T.dot(d1)

        b5 = b5 - d5.mean(axis=0)
        b4 = b4 - d4.mean(axis=0)
        b3 = b3 - d3.mean(axis=0)
        b2 = b2 - d2.mean(axis=0)
        b1 = b1 - d1.mean(axis=0)

    print("----" * 10)
    print("{0:3} error = {1:.5f}".format(epoch, error))

    dz2 = activation_derivative(z1).dot(w2) # dz2 / dz1
    dz3 = activation_derivative(z2).dot(w3) # dz3 / dz2
    dz4 = activation_derivative(z3).dot(w4) # dz4 / dz3
    dz5 = activation_derivative(z4).dot(w5) # dz5 / dz4
    
    as_ = [a1, a2, a3, a4, a5]
    dzs = [dz2, dz3, dz4, dz5]
    ds = [d5, d4, d3, d2, d1]

    return error, ws, bs, as_, dzs, ds, history<jupyter_output><empty_output><jupyter_text>Case 1 - Small Random Number + Sigmoid<jupyter_code>w1 = np.random.randn(64, 100) * 0.05
w2 = np.random.randn(100, 100) * 0.05
w3 = np.random.randn(100, 100) * 0.05
w4 = np.random.randn(100, 100) * 0.05
w5 = np.random.randn(100, 10) * 0.05

b1 = np.zeros((1, 100))
b2 = np.zeros((1, 100))
b3 = np.zeros((1, 100))
b4 = np.zeros((1, 100))
b5 = np.zeros((1, 10))

ws = [w1, w2, w3, w4, w5]
bs = [b1, b2, b3, b4, b5]

num_epoch = 1000
learning_rate = 0.0003

error, ws, bs, as_, dzs, ds, sigmoid_history = train(X, y, num_epoch, learning_rate,
                                                     ws, bs,
                                                     activation=sigmoid,
                                                     activation_derivative=sigmoid_derivative)
plot_w(ws, range_=(-1.0, +1.0))
plot_a(as_, (0.0, 1.0))
plot_dzs(dzs)
plot_ds(ds, range_=(-0.2, +0.2))<jupyter_output><empty_output><jupyter_text>Case 2 - Small Random Number + tanh<jupyter_code>w1 = np.random.randn(64, 100) * 0.05
w2 = np.random.randn(100, 100) * 0.05
w3 = np.random.randn(100, 100) * 0.05
w4 = np.random.randn(100, 100) * 0.05
w5 = np.random.randn(100, 10) * 0.05

b1 = np.zeros((1, 100))
b2 = np.zeros((1, 100))
b3 = np.zeros((1, 100))
b4 = np.zeros((1, 100))
b5 = np.zeros((1, 10))

ws = [w1, w2, w3, w4, w5]
bs = [b1, b2, b3, b4, b5]

num_epoch = 100
learning_rate = 0.0003

error, ws, bs, as_, dzs, ds, sigmoid_history = train(X, y, num_epoch, learning_rate,
                                                     ws, bs,
                                                     activation=tanh,
                                                     activation_derivative=tanh_derivative)
plot_w(ws, range_=(-1.0, +1.0))
plot_a(as_, (-1.0, 1.0))
plot_dzs(dzs)
plot_ds(ds, range_=(-0.2, +0.2))<jupyter_output><empty_output><jupyter_text>Case 3 - Xavior Initialization + tanh<jupyter_code>w1 = np.random.uniform(low=-np.sqrt(6 / (64+100)),  high=np.sqrt(6 / (64+100)), size=(64, 100))
w2 = np.random.uniform(low=-np.sqrt(6 / (100+100)), high=np.sqrt(6 / (100+100)), size=(100, 100))
w3 = np.random.uniform(low=-np.sqrt(6 / (100+100)), high=np.sqrt(6 / (100+100)), size=(100, 100))
w4 = np.random.uniform(low=-np.sqrt(6 / (100+100)), high=np.sqrt(6 / (100+100)), size=(100, 100))
w5 = np.random.uniform(low=-np.sqrt(6 / (100+10)),  high=np.sqrt(6 / (100+10)), size=(100, 10))

b1 = np.zeros((1, 100))
b2 = np.zeros((1, 100))
b3 = np.zeros((1, 100))
b4 = np.zeros((1, 100))
b5 = np.zeros((1, 10))

ws = [w1, w2, w3, w4, w5]
bs = [b1, b2, b3, b4, b5]

num_epoch = 100
learning_rate = 0.0003

error, ws, bs, as_, dzs, ds, sigmoid_history = train(X, y, num_epoch, learning_rate,
                                                     ws, bs,
                                                     activation=tanh,
                                                     activation_derivative=tanh_derivative)
plot_w(ws, range_=(-1.0, +1.0))
plot_a(as_, (-1.0, 1.0))
plot_dzs(dzs)
plot_ds(ds, range_=(-0.2, +0.2))<jupyter_output><empty_output><jupyter_text>Case 4 - Xavior Initialization + ReLU<jupyter_code>w1 = np.random.uniform(low=-np.sqrt(6 / (64+100)),  high=np.sqrt(6 / (64+100)), size=(64, 100))
w2 = np.random.uniform(low=-np.sqrt(6 / (100+100)), high=np.sqrt(6 / (100+100)), size=(100, 100))
w3 = np.random.uniform(low=-np.sqrt(6 / (100+100)), high=np.sqrt(6 / (100+100)), size=(100, 100))
w4 = np.random.uniform(low=-np.sqrt(6 / (100+100)), high=np.sqrt(6 / (100+100)), size=(100, 100))
w5 = np.random.uniform(low=-np.sqrt(6 / (100+10)),  high=np.sqrt(6 / (100+10)), size=(100, 10))

b1 = np.zeros((1, 100))
b2 = np.zeros((1, 100))
b3 = np.zeros((1, 100))
b4 = np.zeros((1, 100))
b5 = np.zeros((1, 10))

ws = [w1, w2, w3, w4, w5]
bs = [b1, b2, b3, b4, b5]

num_epoch = 100
learning_rate = 0.0003

error, ws, bs, as_, dzs, ds, sigmoid_history = train(X, y, num_epoch, learning_rate,
                                                     ws, bs,
                                                     activation=relu,
                                                     activation_derivative=relu_derivative)
plot_w(ws, range_=(-1.0, +1.0))
plot_a(as_)
plot_a(as_, (0.001, 5.0))
plot_dzs(dzs)
plot_ds(ds, range_=(-0.2, +0.2))
plot_ds(ds, exclude=0.001, range_=(-0.2, +0.2))<jupyter_output><empty_output><jupyter_text>Case 4 - He Initialization + ReLU<jupyter_code>w1 = np.random.uniform(low=-np.sqrt(6 / 64),  high=np.sqrt(6 / 64), size=(64, 100))
w2 = np.random.uniform(low=-np.sqrt(6 / 100), high=np.sqrt(6 / 100), size=(100, 100))
w3 = np.random.uniform(low=-np.sqrt(6 / 100), high=np.sqrt(6 / 100), size=(100, 100))
w4 = np.random.uniform(low=-np.sqrt(6 / 100), high=np.sqrt(6 / 100), size=(100, 100))
w5 = np.random.uniform(low=-np.sqrt(6 / 100),  high=np.sqrt(6 / 100), size=(100, 10))

b1 = np.zeros((1, 100))
b2 = np.zeros((1, 100))
b3 = np.zeros((1, 100))
b4 = np.zeros((1, 100))
b5 = np.zeros((1, 10))

ws = [w1, w2, w3, w4, w5]
bs = [b1, b2, b3, b4, b5]

num_epoch = 100
learning_rate = 0.0003

error, ws, bs, as_, dzs, ds, sigmoid_history = train(X, y, num_epoch, learning_rate,
                                                     ws, bs,
                                                     activation=relu,
                                                     activation_derivative=relu_derivative)
plot_w(ws, range_=(-1.0, +1.0))
plot_a(as_)
plot_a(as_, (0.001, 5.0))
plot_dzs(dzs)
plot_ds(ds, range_=(-0.2, +0.2))
plot_ds(ds, exclude=0.001, range_=(-0.2, +0.2))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19121852227654276, 'fraction_numerical': 0.09215017064846416, 'mean_word_length': 2.7750696378830084, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '392750', 'license': 'MIT', 'n_tokens_mistral': 5185, 'n_tokens_neox': 4609, 'n_words': 853, 'path': 'example/01-Weight-Initialization.ipynb', 'repo_name': 'daesookimds/Deep-Learning', 'size': '10841'}