<jupyter_start><jupyter_text>1. Implementing a Feed-Forward-Network<jupyter_code>import numpy as np
from sklearn.datasets import make_moons
import matplotlib.pyplot as plt

X, y = make_moons(n_samples=50, noise=0.2, random_state=42)
plt.scatter(X[:,0],X[:,1], c=y)
# Function to add bias

def add_bias(X):
    X = np.hstack([X, np.ones((X.shape[0], 1))])
    return X

# Overwriting X with added bias

X = add_bias(X)
# Function to calculate the sigmoid activation function

def sigmoid(X):
    return 1 / (1 + np.exp(-X))
a = np.array([-10.0, -1.0, 0.0, 1.0, 10.0])
expected = np.array([0.0, 0.27, 0.5, 0.73, 1.0])
assert np.all(sigmoid(a).round(2) == expected)<jupyter_output><empty_output><jupyter_text>Refer again to the image of network we’re trying to build. What about all the neural connections / arrows? Those are our weights.These weights should also be represented by NumPy arrays, since they will multiplied (vector-matrix multiplication) by input values before getting passed into an activation function.Initialize the network with random weights (for both the hidden and output layers).<jupyter_code>num_rows, num_cols = X.shape
print(num_rows, num_cols)
#w1 = np.random.rand(3,2) #weights hidden
#w2 = np.random.rand(3,1) #weights output

# Writing the function to get random weights

def get_random_weights(n_neurons, n_features):
    """
    Calculates randomized weights for multiple features and neurons 
    from a normal distribution.
    """
    return np.random.normal(size=(n_neurons, n_features))

# Creating weight 1 and weight 2

w1 = get_random_weights(3,2)
w2 = get_random_weights(3,1)
# Writing the feed_forward function

def feed_forward(X,w1,w2):
  ''' 
  STEP 1. calculate the dot product of X (shape of (50,3))
  and the weights of the first layer (shape of (3, 2))
  this results in an output shape of (50, 2)
  '''
  step1 = np.dot(X, w1)


  '''
  STEP 2. apply the sigmoid function on the result
  (applying the sigmoid function on a 50x2 matrix outputs the same shape.
  '''
  step2 = sigmoid(step1)


  '''
  STEP 3. append an extra 1 for the bias to the result
  (this results in a 50x3 matrix)
  '''
  #step3 = np.hstack([step2, np.ones((step2.shape[0], 1))])
  step3 = add_bias(step2)
    

  '''
  STEP 4. calculate the dot product of X (from step 3) (shape of (50, 3))
  and the weights of the second layer (shape of (3, 1)). 
  This results in an output shape of (50, 1)
  '''
  step4 = np.dot(step3, w2)


  '''
  STEP 5. apply the sigmoid function on the result
  '''
  step5 = sigmoid(step4)

    
  '''
  STEP 6. return all intermediate results
  '''
  return step2, step5

feed_forward(X, w1, w2)
# Write a loss function

def log_loss(ytrue, ypred):
    loss = -(ytrue * np.log(ypred) + (1 - ytrue)* np.log(1 - ypred))
    return loss
# Testing the loss function

ytrue = np.array([0.0, 0.0, 1.0, 1.0])
ypred = np.array([0.01, 0.99, 0.01, 0.99])
expected = np.array([0.01, 4.61, 4.61, 0.01])
assert np.all(log_loss(ytrue, ypred).round(2) == expected)<jupyter_output><empty_output><jupyter_text>Getting Initial LossRun your feed-forward function, and try getting an array of log-loss values for each of your data points, compared to the actual labels. Note that you’ll need to reshape the ytrue values from a (N, ) array to a (N, 1) array (e.g. both the prediction and the actual values should be (50,1)). Make sure the shape of the loss values is also (N,1).<jupyter_code># Getting Initial Loss

out1, out2 = feed_forward(X, w1, w2)
hidden_output = out1
yhat = out2
ytrue = y.reshape(-1, 1)
log_loss(ytrue, yhat)<jupyter_output><empty_output><jupyter_text>Writing a Backpropagation Function<jupyter_code>sig_deriv_2 = hidden_output * ( 1 - hidden_output)
sig_deriv_2.shape

ytrue = y.reshape(-1, 1)
loss = log_loss(ytrue , yhat)
error = (yhat - ytrue)# * loss
sig_deriv = yhat * (1 - yhat)
y_grad = sig_deriv * error
error.shape
def backprop(weights,
             out1,
             out2,
             ytrue,
             X,
             LR):

    ytrue = y.reshape(-1, 1)
    wH = weights[0]
    wO = weights[1]

    '''
    EQUATION A: error=(y^−ytrue)×loss
    '''
    loss = log_loss(ytrue , yhat)
    error = (yhat - ytrue) * loss

    '''
    EQUATION B: ∇y=act′(Ho⋅ωo)×error
    '''
    hidden_out_with_bias = add_bias(hidden_output)
    #don't forget the bias!
    sig_deriv = yhat * (1 - yhat)
    #derivative of the sigmoid function with respect to the
    #hidden output * weights
    y_grad = sig_deriv * error

    '''
    EQUATION C: Δωo=−(∇y⋅Ho)×LR
    '''
    #gradient = sig_deriv * error
    #delta_wo = -np.dot(gradient, hidden_out_with_bias) * LR
    delta_wo = -np.dot(y_grad.T, hidden_out_with_bias) * LR

    #and finally, old weights + delta weights -> new weights!
    wO_new = wO + delta_wo.T

    '''
    EQUATION D: ∇H=act′(X⋅ωh)×(∇y⋅ωo)
    '''
    sig_deriv_2 = hidden_output * ( 1 - hidden_output)
    #gradient = gradient.reshape(50, 1)
    H_grad = sig_deriv_2[:,:2]  * np.dot(y_grad, wO[:-1].T)
    #                      ^^ remove bias              ^^ remove bias
    #exclude the bias (last column) of the outer weights,
    #since it is not backpropagated!
    assert H_grad.shape == (50, 2)

    '''
    EQUATION E: Δωh=−(∇H⋅X)×LR
    '''
    delta_wH = -np.dot(H_grad.T, X).T * LR
    wH_new = wH + delta_wH#.T
    #old weights + delta weights -> new weights!

    # new hidden weights, new output weights
    return wH_new, wO_new<jupyter_output><empty_output><jupyter_text>Running the Backpropagation Algorithm<jupyter_code># setup
#X, y = make_moons(n_samples=200, noise=0.1, random_state=42)
#X = add_bias(X)
#y = y.reshape(-1, 1)
#w1 = get_random_weights(3,2)
#w2 = get_random_weights(3,1)
weights = [
   np.random.normal(size=(3, 2)),
   np.random.normal(size=(3, 1))
]

# train

LOSS_VEC = []
LR = 0.1

for i in range(1000):
    out1, out2 = feed_forward(X, weights[0], weights[1])
    yhat = out2
    loss = log_loss(ytrue, yhat).sum()
    LOSS_VEC.append(loss)
    new_weights = backprop(weights, out1, out2, ytrue, X, LR)
    weights = new_weights
    print(out2[1])

plt.plot(LOSS_VEC)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12916939175931982, 'fraction_numerical': 0.03433616742969261, 'mean_word_length': 3.426193921852388, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '634353', 'license': 'MIT', 'n_tokens_mistral': 2421, 'n_tokens_neox': 2276, 'n_words': 657, 'path': 'building_a_neural_network_from_scratch/Feed_Forward_Network.ipynb', 'repo_name': 'pavrmk/deep_learning', 'size': '6116'}