<jupyter_start><jupyter_text>4. Single-layer Neural Network with Pattern images based on Lengyel-Epstein model- X : imges, Z = W * gradient(X) + b- optimizer : Gradient Descent  1) Import Packages<jupyter_code>from PIL import Image 
import numpy as np
import matplotlib.pyplot as plt
import random
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
import math
import sklearn.metrics as metrics<jupyter_output><empty_output><jupyter_text>2) Make Dataset<jupyter_code># Make dataset(144)
x_orig = []
y_orig = np.zeros((1,48))
for i in range(1,145):
    if i <= 48 :
        folder = 0
    elif i <=96 :
        folder = 1
    else:
        folder = 2

    img = Image.open('144/{0}/pattern_{1}.jpg'.format(folder,i)) 
    data = np.array(img)
    x_orig.append(data)

for i in range(1,3):
    y_orig = np.append(y_orig, np.full((1, 48),i), axis = 1)
# Make dataset(360)
x_orig = []
y_orig = np.zeros((1,120))
for i in range(1,361):
    if i <= 120 :
        folder = 0
    elif i <=240 :
        folder = 1
    else:
        folder = 2

    img = Image.open('360/{0}/pattern_{1}.jpg'.format(folder,i)) 
    data = np.array(img)
    x_orig.append(data)

for i in range(1,3):
    y_orig = np.append(y_orig, np.full((1, 120),i), axis = 1)
# Make dataset(720)
x_orig = []
y_orig = np.zeros((1,240))
for i in range(1,721):
    if i <= 240 :
        folder = 0
    elif i <=480 :
        folder = 1
    else:
        folder = 2

    img = Image.open('720/{0}/pattern_{1}.jpg'.format(folder,i)) 
    data = np.array(img)
    x_orig.append(data)

for i in range(1,3):
    y_orig = np.append(y_orig, np.full((1, 240),i), axis = 1)
x_orig = np.array(x_orig)
print(x_orig.shape)
print(y_orig.shape)  
# Random shuffle
s = np.arange(x_orig.shape[0])
np.random.shuffle(s)

x_shuffle = x_orig[s,:]
y_shuffle = y_orig[:,s]

print(x_shuffle.shape)
print(y_shuffle.shape)
# y_shuffle
# Split train and test datasets
x_train_orig, x_test_orig, y_train_orig, y_test_orig = train_test_split(x_shuffle,y_shuffle.T, 
                                                                        test_size=0.3,  shuffle=True, random_state=1004)
print(x_train_orig.shape)
print (y_train_orig.shape)
# Flatten the training and test images
x_train_flatten = x_train_orig.reshape(x_train_orig.shape[0], -1).T
x_test_flatten = x_test_orig.reshape(x_test_orig.shape[0], -1).T

# Normalize image vectors
x_train = x_train_flatten/255.
x_test = x_test_flatten/255.

# Convert training and test labels to one hot matrices
enc = OneHotEncoder()
y1 = y_train_orig.reshape(-1,1)
enc.fit(y1)
y_train = enc.transform(y1).toarray()
y_train = y_train.T

y2 = y_test_orig.reshape(-1,1)
enc.fit(y2)
y_test = enc.transform(y2).toarray()
y_test = y_test.T

# Explore dataset 
print ("number of training examples = " + str(x_train.shape[1]))
print ("number of test examples = " + str(x_test.shape[1]))
print ("x_train shape: " + str(x_train.shape))
print ("y_train shape: " + str(y_train.shape))
print ("x_test shape: " + str(x_test.shape))
print ("y_test shape: " + str(y_test.shape))<jupyter_output>number of training examples = 1008
number of test examples = 432
x_train shape: (4096, 1008)
y_train shape: (3, 1008)
x_test shape: (4096, 432)
y_test shape: (3, 432)<jupyter_text>3) Define required functions<jupyter_code>def initialize_parameters(nx, ny):
    """
    Argument:
        nx -- size of the input layer (4096)
        ny -- size of the output layer (3)
    
    Returns:
        W -- weight matrix of shape (ny, nx)
        b -- bias vector of shape (ny, 1)

    """   
    np.random.seed(1)

    W = np.random.randn(ny,nx)*0.01
    b = np.zeros((ny,1))

    assert(W.shape == (ny, nx))
    assert(b.shape == (ny, 1))

    
    return W, b  
def softmax(Z):
    # compute the softmax activation
    
    S = np.exp(Z + np.max(Z)) / np.sum(np.exp(Z + np.max(Z)), axis = 0)
    
    return S
def classlabel(Z):
    # probabilities back into class labels
    y_hat = Z.argmax(axis=0)
    
    return y_hat
def gradient_vec(X):
    g_X_r = np.gradient(X, axis = 1)
    g_X_c = np.gradient(X, axis = 0)
    g_X = g_X_r**2 + g_X_c**2
    return g_X
def propagate(W, b, X, Y):

    m = X.shape[1]
#     n = Y.shape[0]
    
    # Forward Propagation
    Z = np.dot(W, gradient_vec(X))+ b
    A = softmax(Z)     # compute activation
    
    cost = (-1/m) * np.sum(Y * np.log(A)) # compute cost (Cross_entropy)
    
    # Backward propagation
    dW = (1/m) * (np.dot(gradient_vec(X),(A-Y).T)).T
    db = (1/m) * (np.sum(A-Y))

#     assert(dW.shape == W.shape)
#     assert(db.dtype == float)
#     cost = np.squeeze(cost)
#     assert(cost.shape == (Y.shape[0],1))
    
    grads = {"dW": dW,
             "db": db}
    
    return grads, cost<jupyter_output><empty_output><jupyter_text>4) Single-Gradient Layer Neural Network with Gradient Descent<jupyter_code>def optimize(X, Y, num_iterations, learning_rate, print_cost = False):

    costs = []
    W, b = initialize_parameters(4096,3)
    for i in range(num_iterations):

        grads, cost = propagate(W,b,X,Y)

        dW = grads["dW"]
        db = grads["db"]
        
        W = W - (learning_rate) * dW
        b = b - (learning_rate) * db

        # Record the costs for plotting
        if i % 100 == 0:
            costs.append(cost)
            
        # Print the cost every 100 training iterations
        if print_cost and i % 200 == 0:
            print ("Cost after iteration %i: %f" %(i, cost))
            
    # plot the cost
    plt.plot(costs)
    plt.ylabel('cost')
    plt.xlabel('iterations (per 200)')
    plt.title("Learning rate =" + str(learning_rate))
    plt.show()
    
    # Lets save the trainded parameters in a variable
    params = {"W": W,
              "b": b}    
    grads = {"dW": dW,
             "db": db}
    
    return params, grads, costs
params, grads, costs = optimize(x_train, y_train, num_iterations= 2000, learning_rate = 10, print_cost = True)

print ("W = " + str(params["W"]))
print ("b = " + str(params["b"]))<jupyter_output>Cost after iteration 0: 1.098714
Cost after iteration 200: 0.732172
Cost after iteration 400: 0.602438
Cost after iteration 600: 0.528716
Cost after iteration 800: 0.476622
Cost after iteration 1000: 0.436599
Cost after iteration 1200: 0.404258
Cost after iteration 1400: 0.377234
Cost after iteration 1600: 0.354110
Cost after iteration 1800: 0.333972<jupyter_text>5) Accuracy Analysis<jupyter_code>def predict(W, b, X) :
    '''
    Predict the label(0,1,2) using argmax
    
    Arguments:
    X : data of size (num_px * num_px, number of examples)
    
    Returns:
    y_prediction : predictions (0/1/2) for the examples(my_image)
    '''
    
    m = X.shape[1]
    
    # Compute "A" predicting the probabilities
    Z = np.dot(W, gradient_vec(X))+ b
    A = softmax(Z) 

    # Convert probabilities A to actual predictions
    y_prediction = A.argmax(axis=0)
    
#     assert(y_prediction.shape == (1, m))
    
    return y_prediction
# Predict test/train set
W1 = params['W']
b1 = params['b']
y_prediction_train = predict(W1, b1, x_train)
y_prediction_test = predict(W1, b1, x_test)
print(y_prediction_train)
print(y_prediction_test)
# Print train/test Errors
print("train accuracy : ", metrics.accuracy_score(y_prediction_train, y_train_orig))
print("test accuracy : ", metrics.accuracy_score(y_prediction_test, y_test_orig))<jupyter_output>[2 1 2 ... 0 2 1]
[0 0 0 2 0 2 0 1 1 1 0 1 1 2 1 1 2 0 2 0 2 2 2 2 2 1 1 1 0 1 0 0 2 1 2 2 1
 1 0 2 1 2 1 0 0 1 0 1 1 1 1 1 2 2 2 1 2 0 2 0 2 1 2 2 0 0 2 2 2 1 1 0 1 0
 2 2 0 2 2 1 0 2 2 0 2 0 2 0 0 0 2 1 1 2 2 0 0 1 1 1 0 1 2 2 2 1 2 2 2 2 0
 1 0 1 2 1 1 2 1 0 1 2 1 2 2 0 1 0 1 0 1 2 2 0 2 0 1 2 1 0 1 1 2 1 0 0 1 2
 0 0 1 0 0 0 0 0 1 0 0 2 1 0 1 0 2 1 2 0 1 0 2 1 0 0 0 0 2 2 2 0 0 2 2 2 0
 1 0 0 1 0 0 2 2 1 0 2 0 1 0 1 1 0 0 1 2 2 1 2 1 0 2 2 2 1 2 1 0 2 1 0 1 0
 1 0 1 0 1 0 1 0 2 0 2 2 0 1 2 0 2 2 2 0 0 2 1 2 0 0 0 0 0 2 0 1 1 0 0 2 0
 1 0 1 0 0 0 2 0 0 1 0 2 0 2 1 2 2 1 2 1 1 0 2 1 1 0 1 1 0 0 0 2 2 2 1 1 1
 1 2 0 2 1 2 0 1 0 0 1 0 1 0 2 0 2 0 0 1 0 1 0 1 2 1 0 1 2 2 1 1 1 2 1 2 2
 2 1 0 1 1 1 1 1 1 0 2 1 1 2 0 1 0 2 0 1 0 2 1 1 1 2 0 1 0 1 1 0 2 2 0 1 0
 1 1 0 0 1 0 2 1 1 1 0 1 0 1 2 0 2 1 0 2 2 1 1 1 2 0 2 1 0 0 0 1 2 0 2 0 2
 1 1 1 0 0 0 2 2 2 2 0 2 1 0 2 0 1 0 1 2 0 1 1 1 1]
train accuracy :  0.9553571428571429
test accuracy :  0.3425925925925926
{'fraction_non_alphanumeric': 0.12029176132966639, 'fraction_numerical': 0.09542030371876122, 'mean_word_length': 2.61139896373057, 'pattern_counts': {'":': 6, '<': 24, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 72, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '171714', 'license': 'MIT', 'n_tokens_mistral': 3916, 'n_tokens_neox': 3225, 'n_words': 1125, 'path': 'Classification_of_Pattern_Images/4_1gradient_layer_NN_with_GD.ipynb', 'repo_name': 'OH-Seoyoung/MachineLearning_with_Patterns_Based_on_Lengyel-Epstein_model', 'size': '8363'}