<jupyter_start><jupyter_text>=   Artificial Neurons and Neural NetworksThis notebook includes implementations of artificial neurons (Perceptron, Adaline & Logistic Regression) and simple neural networks from scratch in Python.1) [The Perceptron](artificial-neuron-1-the-perceptron)2) [Adaline](artificial-neuron-2-adaline)3) [Logistic Regression](artificial-neuron-3-logistic-regression)4) [Binary Classification with NLL Loss and ReLu activation](binary-classification-with-nll-loss-and-relu-activation)5) [Binary Classification with NLL Loss and Sigmoid activation](binary-classification-with-nll-loss-and-sigmoid-activation)6) [Multiclass Classification with Cross Entropy Loss](multiclass-classification-with-cross-entropy-loss)7) [Multiclass Classification with Cross Entropy Loss and L2 Regularization](multiclass-classification-with-cross-entropy-loss-and-l2-regularization)<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier

import warnings
warnings.filterwarnings("ignore")
from sklearn.datasets import load_breast_cancer
data=load_breast_cancer(as_frame=True)
X,y=data.data,data.target
# Since the default in the file is 0=malignant 1=benign we want to reverse these
y=(y==0).astype(int)
X,y= np.array(X),np.array(y)

# Let's set aside a test set and use the remainder for training and cross-validation
X_train,X_test,y_train,y_test = train_test_split(X, y, random_state=0,test_size=0.2)

data.data.head()
# Let's scale our data to help speed convergence
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>Artificial Neurons Artificial Neuron 1: The Perceptron<jupyter_code>class Perceptron:
    
    def __init__(self,eta=0.01,n_iter=100,random_state=0):
        self.eta=eta
        self.n_iter=n_iter
        self.random_state=random_state
        self.cost_path=[]
    
    def fit(self,X,y):
        # Initialize the weights and bias (weights[0]) to small random numbers
        rgen=np.random.RandomState(self.random_state)
        self.weights = rgen.normal(loc=0.0,scale=0.01,size=(1+X.shape[1]))
        
        # Train the perceptron using stochastic gradient descent
        for i in range(self.n_iter):
            # Keep track of cost for each iteration
            cost_total = 0
            # Stochastic gradient descent
            for xi,yi in zip(X,y):
                # Get prediction
                yhat = self.predict(xi)
                # Calculate cost function - SSE
                error = yi-yhat
                cost = 0.5*(error)**2
                gradient_weights = error*xi
                gradient_bias = error*1
                # Update rule
                delta_weights = self.eta * gradient_weights
                delta_bias = self.eta * gradient_bias
                # Update the weights
                self.weights[1:] += delta_weights
                # Update bias
                self.weights[0] += delta_bias
                # Count errors
                cost_total+= cost
            #self.error_counts.append(error_count)
            self.cost_path.append(cost_total)
        return self
    
    def predict(self,X):
        z = np.dot(X,self.weights[1:]) + self.weights[0]
        if z >= 0.0:
            yhat = 1
        else:
            yhat = 0
        return yhat
random_state = 123
perceptron_model = Perceptron(eta=0.00001,n_iter=100, random_state=random_state)
perceptron_model.fit(X_train_scaled,y_train)
plt.plot(range(len(perceptron_model.cost_path)),perceptron_model.cost_path)
plt.xlabel('Epoch')
plt.ylabel('Cost')
plt.show()
test_preds = []
for x in X_test_scaled:
    pred = perceptron_model.predict(x)
    test_preds.append(pred)
test_acc = np.sum(test_preds==y_test)/len(y_test)
print('Test set accuracy is {:.3f}'.format(test_acc))<jupyter_output>Test set accuracy is 0.930<jupyter_text>Artificial Neuron 2: Adaline<jupyter_code>class Adaline:
    
    def __init__(self,eta=0.01,n_iter=100,random_state=0):
        self.eta=eta
        self.n_iter=n_iter
        self.random_state=random_state
        self.cost_path=[]
    
    def fit(self,X,y):
        # Initialize the weights and bias (weights[0]) to small random numbers
        rgen=np.random.RandomState(self.random_state)
        self.weights = rgen.normal(loc=0.0,scale=0.01,size=(1+X.shape[1]))
        
        # Train adaline using batch gradient descent
        for i in range(self.n_iter):
            yhat = self.predict(X)
            # Calculate the cost and gradient of cost with respect to weights and bias
            cost = np.sum(0.5 * (y-yhat)**2)
            gradient_weights = X.T.dot(y-yhat)
            gradient_bias = np.sum(y-yhat)
            # Update weights and bias
            delta_weights = self.eta * gradient_weights
            delta_bias = self.eta * gradient_bias
            self.weights[1:] += delta_weights # Update weights
            self.weights[0] += delta_bias # Update bias
            # Add cost to total cost counter
            self.cost_path.append(cost)
        return self
    
    def predict(self,X):
        z = np.dot(X,self.weights[1:]) + self.weights[0]
        yhat = 1*z
        return yhat
adaline_model = Adaline(eta=0.0001,n_iter=100, random_state=random_state)
adaline_model.fit(X_train_scaled,y_train)
plt.plot(range(len(adaline_model.cost_path)),adaline_model.cost_path)
plt.xlabel('Epoch')
plt.ylabel('Cost')
plt.show()
test_preds = adaline_model.predict(X_test_scaled)
test_preds = np.round(test_preds)
test_acc = np.sum(test_preds==y_test)/len(y_test)
print('Test set accuracy is {:.3f}'.format(test_acc))<jupyter_output>Test set accuracy is 0.921<jupyter_text>Artificial Neuron 3: Logistic Regression<jupyter_code>class LogisticRegressionGD:
    
    def __init__(self,eta=0.01,n_iter=100,random_state=0):
        self.eta=eta
        self.n_iter=n_iter
        self.random_state=random_state
        self.cost_path=[]
    
    def fit(self,X,y):
        # Initialize the weights and bias (weights[0]) to small random numbers
        rgen=np.random.RandomState(self.random_state)
        self.weights = rgen.normal(loc=0.0,scale=0.01,size=(1+X.shape[1]))
        
        # Train adaline using batch gradient descent
        for i in range(self.n_iter):
            yhat = self.predict(X)
            # Calculate the cost and gradient of cost with respect to weights and bias
            cost = (-y.dot(np.log(yhat)) - ((1-y).dot(np.log(1-yhat))))
            gradient_weights = -X.T.dot(y-yhat) # Gradient of cost wrt weights
            gradient_bias = -np.sum(y-yhat) # Gradient of cost wrt bias
            
            # Update the weights and bias
            delta_weights = self.eta * gradient_weights
            delta_bias = self.eta * gradient_bias
            self.weights[1:] -= delta_weights # Update weights
            self.weights[0] -= delta_bias # Update bias
            # Add cost to total cost counter
            self.cost_path.append(cost)
        return self
    
    def predict(self,X):
        z = np.dot(X,self.weights[1:]) + self.weights[0]
        yhat = 1. / (1. + np.exp(-np.clip(z,-250,250)))
        return yhat
logreg_model = LogisticRegressionGD(eta=0.001,n_iter=100)
logreg_model.fit(X_train_scaled,y_train)
plt.plot(range(len(logreg_model.cost_path)),logreg_model.cost_path)
plt.xlabel('Epoch')
plt.ylabel('Cost')
plt.show()
test_preds = logreg_model.predict(X_test_scaled)
test_preds = np.round(test_preds)
test_acc = np.sum(test_preds==y_test)/len(y_test)
print('Test set accuracy is {:.3f}'.format(test_acc))<jupyter_output>Test set accuracy is 0.956<jupyter_text>Shallow Neural Networks from Scratch Binary Classification with NLL Loss and ReLu activationShallow neural network for binary classification with one hidden layer and an output layer.  Network uses Negative Log Likelihood cost function and ReLu as the activation function of the hidden layer.<jupyter_code>class NeuralNet_NLL_relu:
    
    def __init__(self,eta=0.05,n_iter=100,hidden_size=100,random_state=0):
        self.eta=eta # Learning rate
        self.n_iter=n_iter # Number of epochs
        self.hidden_size=hidden_size # Number of units in the hidden layer
        self.random_state=random_state
    
    def predict(self,X):
        # Feed data through the neural network to get z scores of output
        z_hidden = np.dot(X, self.W_1) + self.b_1
        hiddenlayer_output = np.maximum(0, z_hidden) # linear combination plus relu
        z_out = np.dot(hiddenlayer_output, self.W_out) + self.b_out # Output layer
        yhat = 1. / (1. + np.exp(-z_out))
        return yhat
    
    def fit(self,X,y):
        epsilon = 1e-10
        N = X.shape[0] # Number of observations
        P = X.shape[1] # Number of features
        
        # Initialize the weights and biases to small random numbers
        self.W_1 = 0.01 * np.random.randn(P,self.hidden_size) # Weights layer 1
        self.b_1 = np.zeros((1,self.hidden_size)) # Biases layer 1
        self.W_out = 0.01 * np.random.randn(self.hidden_size,1) # Weights in output layer
        self.b_out = np.zeros((1,1)) # Bias in output layer
        
        # Train using batch gradient descent
        for i in range(self.n_iter):
            # Forward propagation to get the predictions
            yhat = self.predict(X)
            # Compute the average NLL loss
            cost = 1/N*(-y.dot(np.log(yhat+epsilon)) - \
                ((1-y).dot(np.log(1-yhat+epsilon))))
            if i % 1000 == 0:
                print("iteration {:d}: loss {:.5f}".format(i, cost.item()))
            
            # Backpropagation

            # Calculate the gradient of the average NLL loss with respect to output layer
            #cost function关于z_out求导，即逻辑回归求导
            gradient_z_out = -1/N*(y.reshape(-1,1)-yhat)
            # linear combination plus relu。参考predict部分，z_out关于W_out求导，系数为hiddenlayer_output
            hiddenlayer_output = np.maximum(0, np.dot(X, self.W_1) + self.b_1) 
            gradient_W_out = np.dot(hiddenlayer_output.T,gradient_z_out) # Gradient wrt output layer weights
            gradient_b_out = np.sum(gradient_z_out) # Gradient wrt output layer bias
            
            # Backpropagate to hidden layer output
            gradient_hidden_output = np.dot(gradient_z_out,self.W_out.T) # Hidden layer error
            # Backpropagate back through relu to hidden layer z
            grad_relu = np.ones(shape=gradient_hidden_output.shape) 
            grad_relu[hiddenlayer_output <=0]=0  # Gradient of hidden layer output wrt ReLu
            gradient_z_hidden = gradient_hidden_output * grad_relu # Gradient of hidden layer wrt z_hidden
            # Backpropagate to W_1 and b_1
            gradient_W_1 = np.dot(X.T,gradient_z_hidden) # Gradient wrt weights layer 1
            gradient_b_1 = np.sum(gradient_z_hidden,axis=0) # Gradient wrt biases layer 1
            
            # Update the weights and biases
            self.W_1 -= self.eta * gradient_W_1
            self.b_1 -= self.eta * gradient_b_1
            self.W_out -= self.eta * gradient_W_out
            self.b_out -= self.eta * gradient_b_out
        
        return self
nn_model = NeuralNet_NLL_relu(eta=0.05,n_iter=10000)
nn_model.fit(X_train_scaled,y_train)

# evaluate test set accuracy
yhat = nn_model.predict(X_test_scaled).squeeze()
yhat = (yhat>=0.5).astype(int)
acc = np.sum(yhat.reshape(-1)==y_test)/len(y_test)

print('Test set accuracy: {:.3f}'.format(acc))<jupyter_output>iteration 0: loss 0.69365
iteration 1000: loss 0.05345
iteration 2000: loss 0.04081
iteration 3000: loss 0.03292
iteration 4000: loss 0.02717
iteration 5000: loss 0.02272
iteration 6000: loss 0.01918
iteration 7000: loss 0.01637
iteration 8000: loss 0.01405
iteration 9000: loss 0.01212
Test set accuracy: 0.965<jupyter_text>Binary Classification with NLL Loss and sigmoid activationShallow neural network for binary classification with one hidden layer and an output layer.  Network uses Negative Log Likelihood cost function and sigmoid function as the activation function of the hidden layer.<jupyter_code>class NeuralNet_NLL_sigmoid:
    
    def __init__(self,eta=0.05,n_iter=100,hidden_size=100,random_state=0):
        self.eta=eta # Learning rate
        self.n_iter=n_iter # Number of epochs
        self.hidden_size=hidden_size # Number of units in the hidden layer
        self.random_state=random_state
    
    def predict(self,X):
        ### BEGIN SOLUTION ###
        # Feed data through the neural network to get z scores of output
        z_hidden = np.dot(X, self.W_1) + self.b_1
        hiddenlayer_output = 1. / (1. + np.exp(-z_hidden))
        z_out = np.dot(hiddenlayer_output, self.W_out) + self.b_out # Output layer
        yhat = 1. / (1. + np.exp(-z_out))
        ### END SOLUTION ###
        return yhat
    
    def fit(self,X,y):
        epsilon = 1e-10
        N = X.shape[0] # Number of observations
        P = X.shape[1] # Number of features
        
        # Initialize the weights and biases to small random numbers
        self.W_1 = 0.01 * np.random.randn(P,self.hidden_size) # Weights layer 1
        self.b_1 = np.zeros((1,self.hidden_size)) # Biases layer 1
        self.W_out = 0.01 * np.random.randn(self.hidden_size,1) # Weights in output layer
        self.b_out = np.zeros((1,1)) #Bias in output layer
        
        # Train using batch gradient descent
        for i in range(self.n_iter):
            ### BEGIN SOLUTION ###
            
            # Forward propagation to get the predictions
            yhat = self.predict(X)
            
            # Compute the average NLL loss
            cost = 1/N*(-y.dot(np.log(yhat+epsilon)) - ((1-y).dot(np.log(1-yhat+epsilon))))
            if i % 1000 == 0:
                print("iteration {:d}: loss {:.5f}".format(i, cost.item()))
            
            # Backpropagation
            # Calculate the gradient of the average NLL loss with respect to output layer weights and bias
            gradient_z_out = -1/N*(y.reshape(-1,1)-yhat)
            hiddenlayer_output = 1. / (1. + np.exp(-(np.dot(X, self.W_1) + self.b_1)))
            gradient_W_out = np.dot(hiddenlayer_output.T,gradient_z_out)
            gradient_b_out = np.sum(gradient_z_out)
            
            # Backpropagate to hidden layer output
            gradient_hidden_output = np.dot(gradient_z_out,self.W_out.T)
            # Backpropagate through sigmoid to hidden layer z
            gradient_z_hidden = gradient_hidden_output * (hiddenlayer_output * (1-hiddenlayer_output))
            # Backpropagate to W_1 and b_1
            gradient_W_1 = np.dot(X.T,gradient_z_hidden) # Gradient of cost wrt weights layer 1
            gradient_b_1 = np.sum(gradient_z_hidden,axis=0) # Gradient of cost wrt biases layer 1
            
            # Update the weights and biases
            self.W_1 -= self.eta * gradient_W_1
            self.b_1 -= self.eta * gradient_b_1
            self.W_out -= self.eta * gradient_W_out
            self.b_out -= self.eta * gradient_b_out
        
        ### END SOLUTION ###
        return self
nn_model = NeuralNet_NLL_sigmoid(eta=0.3,n_iter=10000)
nn_model.fit(X_train_scaled,y_train)

# evaluate test set accuracy
yhat = nn_model.predict(X_test_scaled).squeeze()
yhat = (yhat>=0.5).astype(int)
acc = np.sum(yhat.reshape(-1)==y_test)/len(y_test)

print('Test set accuracy: {:.3f}'.format(acc))<jupyter_output>iteration 0: loss 0.68475
iteration 1000: loss 0.05183
iteration 2000: loss 0.04563
iteration 3000: loss 0.04208
iteration 4000: loss 0.03924
iteration 5000: loss 0.03643
iteration 6000: loss 0.03332
iteration 7000: loss 0.02971
iteration 8000: loss 0.02559
iteration 9000: loss 0.02141
Test set accuracy: 0.974<jupyter_text>Multiclass Classification with Cross Entropy LossEven though we only have two classes, we can implement a neural network for multiclass classificaiton using a softmax layer to get the output probabilities.  Since we use a softmax rather than a sigmoid (as we usually do in binary classification), the below implementation uses cross entropy as the loss function (rather than NLL).  An explanation of the math behind the cross entropy loss function and its gradient can be found [here](http://machinelearningmechanic.com/deep_learning/2019/09/04/cross-entropy-loss-derivative.html)<jupyter_code>class NeuralNet_CrossEntropy:
    
    def __init__(self,eta=0.01,n_iter=100,hidden_size=100,random_state=0):
        self.eta=eta # Learning rate
        self.n_iter=n_iter # Number of epochs
        self.hidden_size=hidden_size # Number of units in the hidden layer
        self.random_state=random_state
    
    def fit(self,X,y):
        
        N = X.shape[0] # Number of observations
        P = X.shape[1] # Number of features
        out_classes = len(np.unique(y)) # Number of classes in y
        
        # Initialize the weights to small random numbers
        self.W1 = 0.01 * np.random.randn(P,self.hidden_size)
        self.b1 = np.zeros((1,self.hidden_size))
        self.W2 = 0.01 * np.random.randn(self.hidden_size,out_classes)
        self.b2 = np.zeros((1,out_classes))
        
        # Train using batch gradient descent
        for i in range(self.n_iter):
            
            # Get the predictions
            yhat = self.predict(X)
            
            # Compute the average cross-entropy loss
            # Avg cross-entropy loss: 1/N * (-log(probability of correct class (y)))
            correct_logprobs = -np.log(yhat[range(N),y])
            loss = 1/N*np.sum(correct_logprobs)
            if i % 1000 == 0:
                print("iteration {:d}: loss {:.5f}".format(i, loss))
            
            # Calculate the gradient of the average cross-entropy loss with respect to z_out
            # Gradient = 1/size * (yhat - p(x)), where p(x) is the one-hot encoded vector of y
            # E.g. if y = 0 and 3 classes, p(x)=[1,0,0]. If y=2, p(x) = [0,0,1]
            gradient = yhat.copy()
            gradient[range(N),y] = gradient[range(N),y] - 1
            gradient *= 1/N # Get the average over all observations in batch
            
            # Backpropate the gradient to the parameters
            # First backprop to get gradient with respect to W2 and b2 (weights between hidden layer and output)
            hiddenlayer_output = np.maximum(0, np.dot(X, self.W1) + self.b1) # linear combination plus relu
            dW2 = np.dot(hiddenlayer_output.T, gradient)
            db2 = np.sum(gradient, axis=0, keepdims=True)
            
            # Calculate hidden layer error term
            hidden_error = np.dot(gradient, self.W2.T)
            # Backpropagate the ReLU non-linearity
            hidden_error[hiddenlayer_output <= 0] = 0
            # Backprop to get gradient wrt W1 and b1
            dW1 = np.dot(X.T, hidden_error)
            db1 = np.sum(hidden_error, axis=0, keepdims=True)
            
            # Update the weights and biases
            self.W1 += -self.eta * dW1
            self.b1 += -self.eta * db1
            self.W2 += -self.eta * dW2
            self.b2 += -self.eta * db2
            
        return self
    
    def predict(self,X):
        # Feed data through the neural network to get z scores of output
        hiddenlayer_output = np.maximum(0, np.dot(X, self.W1) + self.b1) # First hidden layer incl. relu
        z_out = np.dot(hiddenlayer_output, self.W2) + self.b2 # Output layer
        # Compute the class probabilities from the z scores using softmax function
        yhat = np.exp(z_out) / np.sum(np.exp(z_out), axis=1, keepdims=True) # Shape: (num_examples,num_classes)
        return yhat


nn_model = NeuralNet_CrossEntropy(eta=0.05,n_iter=20000)
nn_model.fit(X_train_scaled,y_train)

# evaluate test set accuracy
yhat = nn_model.predict(X_test_scaled)
predicted_class = np.argmax(yhat, axis=1)
print('Test set accuracy: {:.3f}'.format(np.mean(predicted_class == y_test)))<jupyter_output>iteration 0: loss 0.69385
iteration 1000: loss 0.04716
iteration 2000: loss 0.03357
iteration 3000: loss 0.02516
iteration 4000: loss 0.01923
iteration 5000: loss 0.01501
iteration 6000: loss 0.01198
iteration 7000: loss 0.00981
iteration 8000: loss 0.00818
iteration 9000: loss 0.00692
iteration 10000: loss 0.00592
iteration 11000: loss 0.00512
iteration 12000: loss 0.00447
iteration 13000: loss 0.00392
iteration 14000: loss 0.00346
iteration 15000: loss 0.00307
iteration 16000: loss 0.00274
iteration 17000: loss 0.00246
iteration 18000: loss 0.00222
iteration 19000: loss 0.00201
Test set accuracy: 0.965<jupyter_text>Multiclass Classification with Cross Entropy Loss and L2 RegularizationImplementation of the above network which includes L2 regularization together with cross entropy as the loss function.<jupyter_code>class NeuralNetwithRegularization:
    
    def __init__(self,eta=0.01,n_iter=100,hidden_size=100,reg=1e-3,random_state=0):
        self.eta=eta # Learning rate
        self.n_iter=n_iter # Number of epochs
        self.hidden_size=hidden_size # Number of units in the hidden layer
        self.reg=1e-3 # Regularization penalty
        self.random_state=random_state
    
    def fit(self,X,y):
        
        N = X.shape[0] # Number of observations
        P = X.shape[1] # Number of features
        out_classes = len(np.unique(y)) # Number of classes in y
        
        # Initialize the weights to small random numbers
        self.W1 = 0.01 * np.random.randn(P,self.hidden_size)
        self.b1 = np.zeros((1,self.hidden_size))
        self.W2 = 0.01 * np.random.randn(self.hidden_size,out_classes)
        self.b2 = np.zeros((1,out_classes))
        
        # Train using batch gradient descent
        for i in range(self.n_iter):
            
            # Get the predictions
            yhat = self.predict(X)
            
            # Compute the total average loss = 1/N * (cross-entropy loss + regularization loss)
            # Cross-entropy loss: (-log(normalized probability of correct class))
            # Regularization loss: 0.5 * reg * sum of weights squared, across all features and classes
            correct_logprobs = -np.log(yhat[range(N),y])
            data_loss = np.sum(correct_logprobs)
            reg_loss = 0.5*self.reg*np.sum(self.W1*self.W1) + 0.5*self.reg*np.sum(self.W2*self.W2)
            loss = 1/N * (data_loss + reg_loss)
            if i % 1000 == 0:
                print("iteration {:d}: loss {:.5f}".format(i, loss))
            
            # Calculate the gradient of the average cross-entropy loss
            # Gradient = 1/size * (yhat - p(x)), where p(x) is the one-hot encoded vector of y
            # E.g. if y = 0 and 3 classes, p(x)=[1,0,0]. If y=2, p(x) = [0,0,1]
            gradient = yhat.copy()
            gradient[range(N),y] = gradient[range(N),y] - 1
            gradient *= 1/N # Get the average over all observations in batch
            
            # Backpropate the gradient to the parameters
            # First backprop to get gradient wrt W2 and b2
            hidden_layer = np.maximum(0, np.dot(X, self.W1) + self.b1) # linear combination plus relu
            dW2 = np.dot(hidden_layer.T, gradient)
            db2 = np.sum(gradient, axis=0, keepdims=True)
            
            # Calculate hidden layer error term
            hidden_error = np.dot(gradient, self.W2.T)
            # Backpropagate the ReLU non-linearity
            hidden_error[hidden_layer <= 0] = 0
            # Backprop to get gradient wrt W1 and b1
            dW1 = np.dot(X.T, hidden_error)
            db1 = np.sum(hidden_error, axis=0, keepdims=True)
            
            # Add regularization gradient contribution to the gradients
            dW2 += 1/N * (self.reg * self.W2)
            dW1 += 1/N * (self.reg * self.W1)
            
            # Update the weights and biases
            self.W1 += -self.eta * dW1
            self.b1 += -self.eta * db1
            self.W2 += -self.eta * dW2
            self.b2 += -self.eta * db2
            
        return self
    
    def predict(self,X):
        # Feed data through the neural network to get z scores of output
        hidden_layer = np.maximum(0, np.dot(X, self.W1) + self.b1) # First hidden layer incl. relu
        out_scores = np.dot(hidden_layer, self.W2) + self.b2 # Output layer
        # Compute the class probabilities from the z scores using softmax function
        yhat = np.exp(out_scores) / np.sum(np.exp(out_scores), axis=1, keepdims=True) # Shape: (num_examples,num_classes)
        return yhat


nn_model = NeuralNetwithRegularization(eta=0.05,reg=1e-3, n_iter=20000)
nn_model.fit(X_train_scaled,y_train)
# evaluate test set accuracy
yhat = nn_model.predict(X_test_scaled)
predicted_class = np.argmax(yhat, axis=1)
print('Test accuracy: {:.3f}'.format(np.mean(predicted_class == y_test)))<jupyter_output>Test accuracy: 0.982
{'fraction_non_alphanumeric': 0.09860053426896855, 'fraction_numerical': 0.03620270324149755, 'mean_word_length': 2.6058079355951698, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '142266', 'license': 'MIT', 'n_tokens_mistral': 8697, 'n_tokens_neox': 7989, 'n_words': 2040, 'path': '1_intro_neuralnets/neural_networks_from_scratch.ipynb', 'repo_name': 'Shunian-Chen/AIPI540-Deep-Learning-Applications', 'size': '25081'}