<jupyter_start><jupyter_text>Exercise H3.2: MLP Regression<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import sympy
import random
%matplotlib inline
rdata = np.loadtxt(r"C:\Users\lcube\Desktop\jupyter\BCCN\MI\given\RegressionData.txt") #load the given regression data
#print(rdata)
## A) Initialization:

N1 = 3 #number of hidden nodes

x = rdata[:,0]
y = rdata[:,1]

w10 = random.uniform(-0.5,0.5) #input weight
b10 = random.uniform(-0.5,0.5) #input bias

def hidden(x): #hidden layer

    w10 = [random.uniform(-0.5,0.5) for _ in range(N1)] #input weight
    
    return (np.tanh((np.dot(w10,x)) - b10))

w21 = random.uniform(-0.5,0.5) #output weight
b21 = random.uniform(-0.5,0.5) #output bias
    
def output(hid): #output layer
    
    w21 = random.uniform(-0.5,0.5) #output weight
    
    return((np.dot(w21,hid)) - b21)

## B) Iterative learning:

# 1. Forward & Back Propagation

# a) Forward Propagation

z = np.linspace(1,len(x),len(x))

hid = [hidden(x[i]) for i in range(len(x))] #hidden layer for each input x
hid = np.array(hid)
print(hid)

#plot the activity of the hidden layer
plt.plot(z,hid)
plt.xlabel('input # (x_i)')
plt.ylabel('hidden layer activity')
plt.show()

out = np.array(output(hid)) #connect each hidden layer neuron to the output
print(out)

#plot the activity of the output layer
plt.plot(z,out)
plt.xlabel('input # (x_i)')
plt.ylabel('output layer activity')
plt.show()

yout = [(out[i][0] + out[i][1] + out[i][2]) for i in range(len(x))] #sum each of the output's activity
print(yout)

#plot the activity of the total output y
plt.plot(z,yout)
plt.xlabel('input # (x_i)')
plt.ylabel('output y')
plt.show()

# b) Output error

def output_error(yout):
    
    qe = 0.5 * (y - yout) ** 2 #quadratic error cost function
    qe = np.sum(qe) #sum over each input to return a single value
    return qe

qe = output_error(yout)
print('output error:', qe)

# c) Backpropagation

def derhidden(x): #hidden layer derivative

    return (1 / (np.cosh(np.dot(w10,x) - b10) ** 2))

def deroutput(hid): #output layer derivative
    
    return(0.5 * (np.dot(w21,hid) - b21) ** 2)

def le_hidden(): #local error hidden
    
    derhid = [derhidden(x[i]) for i in range(len(x))] 
    derhid = np.array(hid)
    
    return derhid

le_hid = np.array(le_hidden())

def le_output(): #local error output
    
    derout = np.array(deroutput(hid))
    deryout = [(derout[i][0] + derout[i][1] + derout[i][2]) for i in range(len(x))]
    deryout = np.array(deryout)
    
    return deryout

le_out = np.array(le_output())
    
#plot the local errors of the hidden layer
plt.plot(z,le_hid)
plt.xlabel('input # (x_i)')
plt.ylabel('hidden layer local errors')
plt.show()

#plot the local error of the output
plt.plot(z,le_out) 
plt.xlabel('input # (x_i)')
plt.ylabel('output local error')
plt.show()

# d) Gradient

#to find the gradient, it will be the 'factor depending on the cost function' -> (y_calculated - y_actual)
#multiplied by the 'factor depending on the model class' -> (local error at neuron * activity of neuron)
#summed over and then divided by 'p'

p = 10 #size of given training set

def grad():
    
    grad_hid = [((yout - y) * (le_hid[:,i] * hid[:,i]) / p) for i in range(N1)] #gradient of hidden, w10
    grad_out = (yout - y) * (le_out * yout) / p #gradient of output, w21
    
    return grad_hid,grad_out

grad = grad()
grad_hid = np.array(grad[0])
grad_out = np.array(grad[1])


print('hidden layer gradients:', grad_hid)
print('output gradients:', grad_out)

# 2. Batch Gradient

def bgrad():

    bgrad_hid = [(-1) * np.sum(grad_hid[i]) for i in range(N1)] #batch gradient of hidden, w10 
    bgrad_out = (-1) * np.sum(grad_out) #batch gradient of output, w21
    
    return bgrad_hid, bgrad_out
    
bgrad = bgrad()
bgrad_hid = bgrad[0]
bgrad_out = bgrad[1]
    
print('hidden layer batch gradient:', bgrad_hid)
print('output batch gradient:', bgrad_out)

# 3. Weight Update

#update the weights according to -> w(t+1) = w(t) + eta * bgrad

eta = 0.5 #learning rate

#training error ->

def training_error(qe):
    
    traine = (qe / p) #output error divided by p
    
    return traine

#print(training_error(qe))

def weight_update(w10,w21):
    
    w10 = w10 + (eta * bgrad_hid)
    w21 = w21 + (eta * bgrad_out)
    
    return w10, w21

wup = weight_update(w10,w21)

w10 = wup[0]
w21 = wup[1]

print('example hidden weights:', w10)
print('example output weights:', w21)

# C) Stopping Criterion

ssv = 10 ** -5 #some small value

tmax = 3000

w10 = [random.uniform(-0.5,0.5) for _ in range(N1)] #input weights
b10 = [random.uniform(-0.5,0.5) for _ in range(N1)] #input bias
w21 = [random.uniform(-0.5,0.5) for _ in range(N1)] #input weights
b21 = [random.uniform(-0.5,0.5) for _ in range(N1)] #input bias   

print('original w10:', w10)
print('original w21:', w21)

def mlp_reg(): #mlp regression
    
    count = 0
    old_te = 0

    hid = np.array([hidden(x[i]) for i in range(len(x))])
    out = np.array(output(hid))
    yout = [(out[i][0] + out[i][1] + out[i][2]) for i in range(len(x))]
        
    qe = output_error(yout)
    
    dte = (training_error(qe) - old_te) #change in training error
    
    while (abs(dte) / training_error(qe)) > ssv: 
    
        count += 1
        
        le_hid = np.array(le_hidden())
        le_out = np.array(le_output())
        
        grad = grad()
        grad_hid = grad[0]
        grad_out = grad[1]
        
        bgrad = bgrad()
        bgrad_hid = bgrad[0]
        bgrad_out = bgrad[1]
        
        weight_update(w10,w21)
        
        if count == tmax:
        
            print('maximum iteration number of', tmax, 'has been reached')
            break 
            

mlp_reg()

print('final w10:', w10)
print('final w21:', w21)<jupyter_output>original w10: [-0.19990453617689785, -0.3246324664293885, -0.08796934089394881]
original w21: [-0.22112792400143644, 0.12519826169835746, -0.2748492669593495]
{'fraction_non_alphanumeric': 0.14831649831649832, 'fraction_numerical': 0.05101010101010101, 'mean_word_length': 3.2741007194244602, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '17751', 'license': 'MIT', 'n_tokens_mistral': 2452, 'n_tokens_neox': 2330, 'n_words': 612, 'path': 'MI1/.ipynb_checkpoints/MI_H3_2-checkpoint.ipynb', 'repo_name': 'lcubelongren/BCCN_classwork', 'size': '5940'}