<jupyter_start><jupyter_text>0. Import Packages<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import scipy.io
import math
import sklearn
import sklearn.datasets


from my_model import forward_prop, backward_prop
from opti_base import init_params, update_params_gd
from opti_mini import random_mini_batches
from opti_mome import init_mome, update_params_mome
from opti_RMSprop import init_rmsp, update_params_rmsp
from opti_adam import init_adam, update_params_adam<jupyter_output><empty_output><jupyter_text>1. Dataset<jupyter_code>np.random.seed(3)
train_X, train_Y = sklearn.datasets.make_moons(n_samples = 300, noise =.2)

# Visualize the data
plt.scatter(train_X[:, 0], train_X[:, 1], c = train_Y, s = 40, cmap = plt.cm.rainbow);
train_X = train_X.T
train_Y = train_Y.reshape((1, train_Y.shape[0]))
print("Shape of train X :", train_X.shape)
print("Shape of train y :", train_Y.shape)<jupyter_output>Shape of train X : (2, 300)
Shape of train y : (1, 300)<jupyter_text>2. Training<jupyter_code>def model(X, Y, layers_dims, optimizer, learning_rate = 0.0007, mini_batch_size = 64, beta = 0.9,
          beta1 = 0.9, beta2 = 0.999,  epsilon = 1e-8, num_epochs = 10000, print_cost = True):
    
    # 3-layer neural network
    
    L = len(layers_dims)            
    costs = []                      
    t = 0     # counter                  
    seed = 10                    
    m = X.shape[1]          
    
    # Initialize parameters
    parameters = init_params(layers_dims, Y.shape[0])

    # Initialize the optimizer
    if optimizer == "gd":
        pass   # no initialization required
    elif optimizer == "momentum":
        v = init_mome(parameters)
    elif optimizer == 'rmsprop':
        s = init_rmsp(parameters)
    elif optimizer == "adam":
        v, s = init_adam(parameters)
    
    for i in range(num_epochs):
        
        seed = seed + 1 # reshuffle differently the dataset after each epoch
        minibatches = random_mini_batches(X, Y, mini_batch_size, seed)
        cost_total = 0
        
        for minibatch in minibatches:
            (minibatch_X, minibatch_Y) = minibatch

            # Forward prop
            a3, caches = forward_prop(minibatch_X, parameters)
            logprobs = np.multiply(-np.log(a3),minibatch_Y) + np.multiply(-np.log(1 - a3), 1 - minibatch_Y)
            cost_total += np.sum(logprobs)

            # Backward prop
            grads = backward_prop(minibatch_X, minibatch_Y, caches)

            # Update parameters
            if optimizer == "gd":
                parameters = update_params_gd(parameters, grads, learning_rate)
            elif optimizer == "momentum":
                parameters, v = update_params_mome(parameters, grads, v, beta, learning_rate)
            elif optimizer == 'rmsprop':
                parameters, s = update_params_rmsp(parameters, grads, s, beta, learning_rate, epsilon)
            elif optimizer == "adam":
                t = t + 1 # counter
                parameters, v, s = update_params_adam(parameters, grads, v, s,
                                                               t, learning_rate, beta1, beta2,  epsilon)
        cost_avg = cost_total / m
        
        if print_cost and i % 1000 == 0:
            print ("Cost after epoch %i: %f" %(i, cost_avg))
        if print_cost and i % 100 == 0:
            costs.append(cost_avg)
                
    plt.plot(costs)
    plt.ylabel('cost')
    plt.xlabel('epochs (per 100)')
    plt.title("Learning rate = " + str(learning_rate))
    plt.show()

    return parameters
def predict(X, y, parameters):

    m = X.shape[1]
    p = np.zeros((1,m), dtype = np.int)
    
    a3, caches = forward_prop(X, parameters)
    
    # prediction (0,1)
    for i in range(0, a3.shape[1]):
        if a3[0,i] > 0.5:
            p[0,i] = 1
        else:
            p[0,i] = 0

    print("Accuracy: "  + str(np.mean((p[0,:] == y[0,:]))))
    
    return p, a3<jupyter_output><empty_output><jupyter_text>- mini-batch Gradient descent<jupyter_code>%%time
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer = "gd")

predictions_gd, prob_gd = predict(train_X, train_Y, parameters)<jupyter_output>Cost after epoch 0: 0.710510
Cost after epoch 1000: 0.655160
Cost after epoch 2000: 0.610186
Cost after epoch 3000: 0.576960
Cost after epoch 4000: 0.555781
Cost after epoch 5000: 0.538643
Cost after epoch 6000: 0.521290
Cost after epoch 7000: 0.503738
Cost after epoch 8000: 0.487588
Cost after epoch 9000: 0.462225<jupyter_text>- Momentum<jupyter_code>%%time
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer = "momentum")

# Predict
predictions_gd, prob_adam = predict(train_X, train_Y, parameters)<jupyter_output>Cost after epoch 0: 0.710531
Cost after epoch 1000: 0.655257
Cost after epoch 2000: 0.610291
Cost after epoch 3000: 0.577044
Cost after epoch 4000: 0.555835
Cost after epoch 5000: 0.538703
Cost after epoch 6000: 0.521370
Cost after epoch 7000: 0.503821
Cost after epoch 8000: 0.487664
Cost after epoch 9000: 0.462354<jupyter_text>- RMSprop<jupyter_code>%%time
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, beta = 0.999, learning_rate = 0.007, optimizer = "rmsprop")

# Predict
predictions_gd, prob_adam = predict(train_X, train_Y, parameters)<jupyter_output>Cost after epoch 0: 0.685336
Cost after epoch 1000: 0.712865
Cost after epoch 2000: 0.707887
Cost after epoch 3000: 0.727836
Cost after epoch 4000: 0.705065
Cost after epoch 5000: 0.717889
Cost after epoch 6000: 0.733250
Cost after epoch 7000: 0.740430
Cost after epoch 8000: 0.730853
Cost after epoch 9000: 0.712207<jupyter_text>- ADAM<jupyter_code>%%time
layers_dims = [train_X.shape[0], 5, 2, 1]
parameters = model(train_X, train_Y, layers_dims, optimizer = "adam")

# Predict
predictions_gd, prob_adam = predict(train_X, train_Y, parameters)<jupyter_output>Cost after epoch 0: 0.710141
Cost after epoch 1000: 0.164173
Cost after epoch 2000: 0.140347
Cost after epoch 3000: 0.136454
Cost after epoch 4000: 0.134194
Cost after epoch 5000: 0.131212
Cost after epoch 6000: 0.130286
Cost after epoch 7000: 0.129598
Cost after epoch 8000: 0.129076
Cost after epoch 9000: 0.128643
{'fraction_non_alphanumeric': 0.10331632653061225, 'fraction_numerical': 0.08689413265306123, 'mean_word_length': 3.057567917205692, 'pattern_counts': {'":': 6, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '170429', 'license': 'MIT', 'n_tokens_mistral': 2582, 'n_tokens_neox': 2161, 'n_words': 598, 'path': 'Without_DL_framework/main_sklearn_moon_dataset.ipynb', 'repo_name': 'OH-Seoyoung/ML_Optimization_Methods', 'size': '6272'}