<jupyter_start><jupyter_text>Implementing an Artificial Neural Network with Mathematical Formulas Only (No Frameworks)  Python Version<jupyter_code>from platform import python_version
print('Python version of Jupyter Notebook:', python_version())

# To update the package, execute the code below in the terminal or comand prompt:
# pip install -U nome_pacote

# To install the exact version of a package, execute the code below on terminal or comand prompt:
# pip install pack_name==desired_version

# After it, reboot jupyter notebook.<jupyter_output>Python version of Jupyter Notebook: 3.8.10<jupyter_text>Import the libraries<jupyter_code># For now only NumPy
import numpy as np<jupyter_output><empty_output><jupyter_text>1st Step - Coding the Functions  [1A - Forward Propagation](https://arxiv.org/pdf/1905.07490.pdf)  Function to Initialize Weights<jupyter_code># Function for random initialization of model parameters
def initial_param(dims_input_layer):
    
    # Dictionary for parameters
    parameters = {}
    
    # Length of layer dimensions
    leng = len(dims_input_layer)
    
    # Length Loop
    for i in range(1, leng):
        
        # Initialization of the weight matrix
        parameters["W" + str(i)] = np.random.randn(dims_input_layer[i], dims_input_layer[i - 1]) * 0.01
        
        # Bias Initialization
        parameters["b" + str(i)] = np.zeros((dims_input_layer[i], 1))
    
    return parameters<jupyter_output><empty_output><jupyter_text>Developing the Sigmoid Function<jupyter_code># Sigmoide Function
def sigmoid(Z):
    A = 1 / (1 + np.exp(-Z))
    return A, Z<jupyter_output><empty_output><jupyter_text>Developing the ReLU Function<jupyter_code># ReLu Function (Rectified Linear Unit)
def relu(Z):
    A = abs(Z * (Z > 0))
    return A, Z<jupyter_output><empty_output><jupyter_text>Develpping the Linear Activation<jupyter_code># Activation Operation
# A is the array with the input data
# W is the weight matrix
# b is the bias
def linear_activation(A, W, b):
    Z = np.dot(W, A) + b
    cache = (A, W, b)
    return Z, cache<jupyter_output><empty_output><jupyter_text>Developing the Forward Propagation Process<jupyter_code># Forward Movement 
def forward(A_prev, W, b, activation):
    
    # If the Sigmoid activation function is chosen
    if activation == "sigmoid":
        Z, linear_cache = linear_activation(A_prev, W, b)
        A, activation_cache = sigmoid(Z)
        
    # If the Relu activation function is chosen    
    elif activation == "relu":
        Z, linear_cache = linear_activation(A_prev, W, b)
        A, activation_cache = relu(Z)
        
    cache = (linear_cache, activation_cache)
    
    return A, cache<jupyter_output><empty_output><jupyter_text>Combining Activation and Propagation<jupyter_code># Foward Propagation
def forward_propagation(X, parameters):
    
    # List of previous values (cache)
    caches = []
    
    # Input Data
    A = X
    
    # Parameter Length
    L = len(parameters) // 2
   
    # Loop
    for i in range(1, L):
      
        # Saves the previous value of A
        A_prev = A
        
        # Forward execution
        A, cache = forward(A_prev, parameters["W" + str(i)], parameters["b" + str(i)], activation = "relu")
        
        # Saves the cache
        caches.append(cache)
    
    # Last layer output
    A_last, cache = forward(A, parameters["W" + str(L)], parameters["b" + str(L)], activation = "sigmoid")
    
    # Saves cache
    caches.append(cache)
    
    return(A_last, caches)<jupyter_output><empty_output><jupyter_text>Developing the Cost Function<jupyter_code># Cost Function (or Error Function)
def calculate_cost(A_last, Y):
    
    # Adjusts the shape of Y to obtain its length (total number of elements)
    m = Y.shape[1]
    
    # Calculate the cost comparing the real and predicted values
    cost = (-1 / m) * np.sum((Y * np.log(A_last)) + ((1 - Y) * np.log(1 - A_last)))
    
    # Adjust cost shape
    custo = np.squeeze(cost)
    
    return(cost)<jupyter_output><empty_output><jupyter_text>1B - Backward Propagation  Developing the Backward Propagation - Sigmoid Backward Function<jupyter_code># Backpropagation Sigmoid Function
# Calculate of the derivative because we don't want the complete value of the function, but its variation
def sigmoid_backward(da, Z):
    
    # Calculate the Z derivative
    dg = (1 / (1 + np.exp(-Z))) * (1 - (1 / (1 + np.exp(-Z))))
    
    # Finding the change in z derivative
    dz = da * dg
    return dz

# Compare with the sigmoid function of forward propagation
# A = 1 / (1 + np.exp(-Z))<jupyter_output><empty_output><jupyter_text>Developing the Backward Propagation - Backward Relu Function<jupyter_code># Relu Function for backpropagation
# Calculate the derivative because we don't want the complete value of the function, but its variation
def relu_backward(da, Z):
    
    dg = 1 * ( Z >= 0)
    dz = da * dg
    return dz<jupyter_output><empty_output><jupyter_text>Developing the Backward Propagation - Linear Activation Backward Function<jupyter_code># Linear activation for backpropagation 
def linear_backward_function(dz, cache):
    
    # Receives the cache (memory)
    A_prev, W, b = cache
    
    # M Shape 
    m = A_prev.shape[1]
    
    # Calculate the derivative of W (result of the operation with dz)
    dW = (1 / m) * np.dot(dz, A_prev.T)
    
    # Calculate the derivative of b (result of the operation with dz)
    db = (1 / m) * np.sum(dz, axis = 1, keepdims = True)
    
    # Calculate the operation derivative
    dA_prev = np.dot(W.T, dz)
    
    return dA_prev, dW, db<jupyter_output><empty_output><jupyter_text>Developing the Backward Propagation - Ativação Linear Backward Function<jupyter_code># Function which defines the activation function (Relu or Sigmoid)
def linear_activation_backward(dA, cache, activation):
    
    # Extract cache
    linear_cache, activation_cache = cache
    
    # Verifica se a ativação é relu
    if activation == "relu":
        dZ = relu_backward(dA, activation_cache)
        dA_prev, dW, db = linear_backward_function(dZ, linear_cache)
        
    # Check if activation is Sigmoid
    if activation == "sigmoid":
        dZ = sigmoid_backward(dA, activation_cache)
        dA_prev, dW, db = linear_backward_function(dZ, linear_cache)
        
    return dA_prev, dW, db<jupyter_output><empty_output><jupyter_text>Combining Activation and backpropagation - Backpropagation Algorithm<jupyter_code># Backpropagation algorithm (calculates the gradients for the update of the weights)
# EV = Forward's Expected Value
# RV = Real Value
def backward_propagation(EV, RV, caches):
    
    # Gradients's Dictionary
    grads = {}
    
    # Datas length (in the cache)
    L = len(caches)
    
    # Extracts the length for the value of m
    m = EV.shape[1]
    
    # Adjust the Y shape
    RV = RV.reshape(EV.shape)
    
    # Calculate the derivative of the final prediction of the network (done at the end of the forward propagation)
    dAL = -((RV / EV) - ((1 - RV) / (1 - EV)))
    
    # Capture the current cache
    current_cache = caches[L - 1]
    
    # Generate a list of gradients for the data, the weights and the bias (final fase of the forward propagation)
    grads["dA" + str(L - 1)], grads["dW" + str(L)], grads["db" + str(L)] = linear_activation_backward(dAL, current_cache, activation = "sigmoid")
    
    #Loop to calculate the derivative during the linear activations with the Relu
    for l in reversed(range(L - 1)):
        
        # Current Cache 
        current_cache = caches[l]
        
        # Derivative calculation
        dA_prev, dW, db = linear_activation_backward(grads["dA" + str(l + 1)], current_cache, activation = "relu")
        
        #Update the gradients dictionary, using the index
        grads["dA" + str(l)] = dA_prev
        grads["dW" + str(l + 1)] = dW
        grads["db" + str(l + 1)] = db
        
    return grads<jupyter_output><empty_output><jupyter_text>Gradient and Weight Update<jupyter_code># Weight Update Function
def weight_update(parameters, grads, learning_rate):
    
    # Length of the structure with the parameters (weights and bias)
    L = len(parameters)//2
    
    # Loop for the weights update
    for l in range(L):
        
        # Weight update
        parameters["W" + str(l + 1)] = parameters["W" + str(l + 1)] - (learning_rate * grads["dW" + str(l + 1)])
        
        # Bias update
        parameters["b" + str(l + 1)] = parameters["b" + str(l + 1)] - (learning_rate * grads["db" + str(l + 1)])
    
    return parameters<jupyter_output><empty_output><jupyter_text>Implementing the Complete Network<jupyter_code># Complete Neural Network Model
def modelNN(X, Y, input_layers_dim, learning_rate = 0.0075, num_iterations = 100):
    
    # List to receive the cost per epoch of training
    costs = []
    
    # Parameters initialization
    parameters = initial_param(input_layers_dim)
    
    # Loop of iterations (epochs)
    for i in range(num_iterations):
        
        # Forward Propagation
        AL, caches = forward_propagation(X, parameters)
        
        # Cost Calculation
        cost = calculate_cost(AL, Y)
        
        # Backward Propagation
        gradient = backward_propagation(AL, Y, caches)
        
        # Weight Update
        parameters = weight_update(parameters, gradient, learning_rate)
        
        # Print of the values of intermetiate cost (cost reduction indicates model learning)
        if i % 10 == 0:
            print("Cost after " + str(i) + " interactions is " + str(cost))
            costs.append(cost)
            
    return parameters, costs
# Predictions Function
def predict(X, parametros):
    AL, caches = forward_propagation(X, parametros)
    return AL<jupyter_output><empty_output><jupyter_text>Step 2 - Training the Artificial Neural Network to Predict Cancer Occurrence<jupyter_code># Imports
import sklearn
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
# Packages Version
%reload_ext watermark
%watermark -a "João Salero" --iversions<jupyter_output>Author: <NAME>

matplotlib: 3.3.4
ipykernel : 5.3.4
sklearn   : 0.24.2
sys       : 3.8.10 (default, May 19 2021, 13:12:57) [MSC v.1916 64 bit (AMD64)]
numpy     : 1.19.5
pandas    : 1.3.0<jupyter_text>Loading DataFrom: https://scikit-learn.org/stable/datasets/index.htmlbreast-cancer-dataset<jupyter_code># Loading
temp = load_breast_cancer()
# Object type
type(temp)
# View the object
temp
# Loading the dataset
dados = pd.DataFrame(columns = load_breast_cancer()["feature_names"], data = load_breast_cancer()["data"])
# Shape
dados.shape
# Data view
dados.head()
# Missing Values Vefiring
dados.isnull().any()
# Target Variable Separation
target = load_breast_cancer()["target"]
type(target)
# Variable Target
target
# Total records by class - Benign Cancer
np.count_nonzero(target == 1)
# Total records by class - Malignant Cancer
np.count_nonzero(target == 0)<jupyter_output><empty_output><jupyter_text>Labels Encoder<jupyter_code># Labels dictionary
labels = {}

# Target variable class names
target_names = load_breast_cancer()["target_names"]

# Mapping
for i in range(len(target_names)):
    labels.update({i:target_names[i]})
# Labels view
labels
# Preparing the predictor variables in X
X = np.array(dados)
# View input data
X
# Spliting the input and output data into training and testing
X_train, X_test, y_train, y_test = train_test_split(X, target, test_size = 0.15, shuffle = True)
# Trainig data Shape
print(X_train.shape)
print(y_train.shape)
# Test data Shape
print(X_test.shape)
print(y_test.shape)
# Adjust the shape of the input data
X_train = X_train.T
X_test = X_test.T
print(X_train.shape)
print(X_test.shape)
# Adjusting the output data
y_train = y_train.reshape(1, len(y_train))
y_test = y_test.reshape(1, len(y_test))
print(y_train.shape)
print(y_test.shape)
# Variables with the input dimensions for the number of neurons
input_layers_dim = [X_train.shape[0], 50, 20, 5, 1]            
input_layers_dim
# Model Training

print("\nStarting Training.\n")

parameters, cost = modelNN(X = X_train, 
                             Y = y_train, 
                             input_layers_dim = input_layers_dim, 
                             num_iterations = 3000, 
                             learning_rate = 0.0075)

print("\nTraining Completed.\n")
# Error plot during training
plt.plot(cost)
# Predictions with training data
y_pred_train = predict(X_train, parameters)
# Predictions view
y_pred_train
# Adjusting the shape in training
y_pred_train = y_pred_train.reshape(-1)
y_train = y_train.reshape(-1)
y_pred_train > 0.5
# Convert predictions to class binary value (0 or 1, using the probability value of 0.5 as threshold)
y_pred_train = 1 * (y_pred_train > 0.5)
y_pred_train
# Calculating accuracy by comparing actual value with predicted value
acc_train = sum(1 * (y_pred_train == y_train)) / len(y_pred_train) * 100
print("Accuracy in training data: " + str(acc_train))
print(classification_report(y_train, y_pred_train, target_names = ['Malignant', 'Benign']))
# Predictions with the model using test data
y_pred_test = predict(X_test, parameters)
# View Data
y_pred_test
# Shapes settings
y_pred_test = y_pred_test.reshape(-1)
y_test = y_test.reshape(-1)
# Converting predictions to class binary value (0 or 1, using the probability value of 0.5 as threshold)
y_pred_test = 1 * (y_pred_test > 0.5)
# View predictions    
y_pred_test
# Calculating the accuracy
accuracy = sum(1 * (y_pred_test == y_test)) / len(y_pred_test) * 100
print("Test data accuracy: " + str(accuracy))
print(classification_report(y_test, y_pred_test, target_names = ['Malignant', 'Benign']))<jupyter_output>precision    recall  f1-score   support

   Malignant       0.93      0.87      0.90        30
      Benign       0.93      0.96      0.95        56

    accuracy                           0.93        86
   macro avg       0.93      0.92      0.92        86
weighted avg       0.93      0.93      0.93        86
{'fraction_non_alphanumeric': 0.09560687130385807, 'fraction_numerical': 0.015066178541255984, 'mean_word_length': 3.3058502576538347, 'pattern_counts': {'":': 4, '<': 75, '<?xml version=': 0, '>': 80, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '178044', 'license': 'MIT', 'n_tokens_mistral': 4811, 'n_tokens_neox': 4576, 'n_words': 1344, 'path': 'Artifitial_Neural_Net_Math/Artifitial_Neural_Net.ipynb', 'repo_name': 'JoaoSalero-AI/Deep-Learning-Study-Repository', 'size': '14209'}