<jupyter_start><jupyter_text>Please read this blog first: [Blog](https://rakesh-malviya.github.io/blog/2017/09/17/4-neural-networks-part-3-feedforward-neural-network.html)<jupyter_code>from sklearn.metrics import accuracy_score,mean_squared_error
from sklearn.model_selection import train_test_split
import math
import numpy as np<jupyter_output><empty_output><jupyter_text>Define activations  SigmoidSigmoid function, $$f(x) = \frac{1}{1+e^{-x}}$$derivative of sigmoid, $$\frac{d}{dx} f(x) = f(x) (1 - f(x))$$<jupyter_code>import numpy as np
class Sigmoid:
    def __init__(self):
        def sig(x):
            return 1 / (1 + math.exp(-x))

        def sig_grad(x):
            sigx = sig(x)
            return sigx*(1-sigx)

        self.func_vec = np.vectorize(sig)
        self.func_vec_grad = np.vectorize(sig_grad)


    def eval(self,z):
        return self.func_vec(z)

    def eval_grad(self,z):
        return self.func_vec_grad(z)<jupyter_output><empty_output><jupyter_text>TanhTanh function, $$f(x) = \frac{e^{x}-e^{-x}}{e^{x}+e^{-x}} = \frac{sinh(x)}{cosh(x)}$$derivative of Tanh, $$\frac{d}{dx} f(x) = 1 - (f(x))^2$$<jupyter_code>class Tanh:
    def __init__(self):
        def tanh(x):
            return math.tanh(x)

        def tanh_grad(x):
            tanhx = tanh(x)
            return (1- tanhx*tanhx)

        self.func_vec = np.vectorize(tanh)
        self.func_vec_grad = np.vectorize(tanh_grad)

    def eval(self, z):
        return self.func_vec(z)

    def eval_grad(self, z):
        return self.func_vec_grad(z)<jupyter_output><empty_output><jupyter_text>def eval_z(self,input_h):$\mathbf{z}^l = \mathbf{W}^l \otimes \mathbf{h}^{l-1} + \mathbf{b}^l$where, $\mathbf{h}^{l-1}$ is input_h in codegrad_act_z is, $\sigma'({\mathbf{z}^l})$  def eval_h(self,input_h):$\mathbf{h}^l = \sigma(\mathbf{z}^l)$  def eval_delta_back(self,grad_act_z_back):$\pmb{\delta}^l = ((\mathbf{W}^{l+1})^T \otimes \pmb{\delta}^{l+1} )\odot  \sigma'({\mathbf{z}^l})$, where in code1. $\mathbf{W}^{l+1}$ is self.W2. $\pmb{\delta}^{l+1}$ is self.dalta3. $\sigma'({\mathbf{z}^l})$ is grad_act_z_back  def accum_grad(self,input_h):$\nabla_{\mathbf{b}^l}J =  \pmb{\delta}^l$ $\nabla_{\mathbf{W}^l}J =  \pmb{\delta}^l \otimes (\mathbf{h}^{l-1})^T$ def update_w(self,lr):$\mathbf{b}_l = \mathbf{b}_l -\eta \cdot \frac{1}{m} \cdot \sum_{i=1}^{m} \nabla_{\mathbf{b}^l}J^{(i)} $$\mathbf{W}_l = \mathbf{W}_l - \eta \cdot \frac{1}{m} \cdot \sum_{i=1}^{m} \nabla_{\mathbf{W}^l}J^{(i)} $where in code1. $\nabla_{\mathbf{b}^l}J$ is self.grad_w2. $\nabla_{\mathbf{b}^l}J$ is self.grad_b3. $\pmb{\delta}^l$ i self.delta4. $\mathbf{h}^{l-1}$ i input_h<jupyter_code>class Dense:
    def __init__(self,layer_size,input_size,act_func=None):
        self.layer_size = layer_size
        self.input_size = input_size
        self.W = np.random.normal(loc=0,scale=1.0,size=(self.layer_size,self.input_size))
        self.b = np.zeros(shape=(self.layer_size,1)) + 0.1
        self.h = None #output
        self.z = None
        self.grad_act_z = None
        self.delta = None
        self.grad_w = np.zeros(shape=self.W.shape)
        self.grad_b = np.zeros(shape=self.b.shape)
        self.grad_w_count = 0
        self.grad_b_count = 0


        if act_func!=None:
            self.act_func = act_func
        else:
            self.act_func = Tanh()

    def eval_z(self,input_h):

        #Test input shape
        if input_h.shape[0]!=self.input_size or input_h.shape[1]!=1:
            raise ValueError("input shape : %s expected (%d,1)"
                             %(str(input_h.shape),self.input_size))

        self.z = np.dot(self.W,input_h) + self.b

        # Test z shape
        if self.z.shape[0] != self.layer_size or self.z.shape[1] != 1:
            raise ValueError("input shape : %s expected (%d,1)"
                             % (str(self.z.shape), self.layer_size))

        self.grad_act_z = self.act_func.eval_grad(self.z)

    def eval_h(self,input_h):
        self.eval_z(input_h)
        self.h = self.act_func.eval(self.z)

    def set_delta(self,delta):
        self.delta = delta

    """
    delta^(l-1) = [ (W^l)^T delta^l ] grad(act(z^(l-1)))
    """
    def eval_delta_back(self,grad_act_z_back):
        if self.delta is None:
            raise ValueError("self.delta is None")

        if self.grad_act_z is None:
            raise ValueError("self.grad_act_z is None")

        return np.matmul(np.transpose(self.W),self.delta) * grad_act_z_back

    def accum_grad(self,input_h):
        self.grad_w += np.matmul(self.delta, np.transpose(input_h))
        self.grad_b += self.delta
        self.grad_w_count += 1
        self.grad_b_count += 1


    def update_w(self,lr):
        self.W = self.W - self.grad_w*lr/self.grad_w_count
        self.b = self.b - self.grad_b*lr/self.grad_b_count
        self.grad_w = np.zeros(shape=self.W.shape)
        self.grad_b = np.zeros(shape=self.b.shape)
        self.grad_w_count = 0
        self.grad_b_count = 0

class NeuralNetwork:
    def __init__(self,input_size,layers,lr=0.01,batch_size=10):
        self.input_size = input_size
        self.lr = lr
        self.layers = []
        self.train_step_count = 0
        self.batch_size = batch_size

        cur_input_size = self.input_size
        for cur_layer_size in layers[:-1]:
            self.layers.append(Dense(cur_layer_size,cur_input_size))
            cur_input_size = cur_layer_size


        self.layers.append(Dense(layers[-1],cur_input_size,act_func=Sigmoid()))

        print([x.layer_size for x in self.layers])


    def forward_step(self,input_x):
        cur_input_h = input_x
        # print(self.layers[1].W)
        for layer in self.layers:
            layer.eval_h(cur_input_h)
            cur_input_h = layer.h

        return cur_input_h

    def backprop_step(self,input_x):
        rev_layer_idxes = list(range(len(self.layers)))[:-1]
        rev_layer_idxes.reverse()
        # print(rev_layer_idxes)
        # backpropagation
        for layer_idx in rev_layer_idxes:
            prev_layer = self.layers[layer_idx+1]
            layer = self.layers[layer_idx]
            prev_layer.accum_grad(layer.h)
            cur_delta = prev_layer.eval_delta_back(layer.grad_act_z)
            layer.set_delta(cur_delta)

        layer.accum_grad(input_x)

    def train_step(self,input_x,y):
        #forward step
        pred_y = self.forward_step(input_x=input_x)

        #Cost grading
        y = np.array(y)
        y = y.reshape((-1,1))
        cur_delta = (pred_y-y)*self.layers[-1].grad_act_z
        self.layers[-1].set_delta(cur_delta)

        #backpropagation step
        self.backprop_step(input_x)
        self.train_step_count+=1
        if self.train_step_count%self.batch_size==0:
            self.updateW()


    def updateW(self):
        for layer in self.layers:
            layer.update_w(self.lr)

    def predict(self,X_train,y_train):
        pred_y = []
        pred_pro_y = []
        for j in range(X_train.shape[0]):
            X = X_train[j, :]
            X = X.reshape((-1, 1))
            y = y_train[j]
            y = y.reshape((-1, 1))
            y_p = self.forward_step(X)
            pred_pro_y.append(y_p[0])
            if y_p[0] > 0.5:
                pred_y.append(1)
            else:
                pred_y.append(0)

        pred_y = np.array(pred_y)
        pred_pro_y = np.array(pred_pro_y)

        return pred_y,pred_pro_y


    def train(self,epoch,X_train,y_train,X_test=None,y_test=None,
              monitor_train_acc=False,monitor_test_acc=False,monitor_checkpoint=10):
        for iepo in range(epoch):
            for j in range(X_train.shape[0]):
                X = X_train[j, :]
                X = X.reshape((-1, 1))
                y = y_train[j]
                y = y.reshape((-1, 1))
                self.train_step(X, y)


            if (iepo % monitor_checkpoint == 0):
                if monitor_train_acc and X_train is not None and y_train is not None:
                    pred_y, pred_pro_y = self.predict(X_train,y_train)
                    print("%d Train acc:%f mse:%f lr:%f" % (
                    iepo, 100.0*accuracy_score(y_train, pred_y), mean_squared_error(y_train, pred_pro_y), self.lr))

                if monitor_test_acc and X_test is not None and y_test is not None:
                    pred_y, pred_pro_y = self.predict(X_test, y_test)
                    print("%d Test acc:%f mse:%f lr:%f" % (
                    iepo, 100.0*accuracy_score(y_test, pred_y), mean_squared_error(y_test, pred_pro_y), self.lr))<jupyter_output><empty_output><jupyter_text>Code to Generate two-spiral dataset<jupyter_code>"""
From tensorflow playground code
"""

def gen2Spiral(sample=500,noise=0,addSin=False):
    n = sample/2
    Xdata = []
    Ydata = []

    def genSpiral(deltaT,label):
        for i in range(n):
            r = float(i)/n*5.0
            t = 1.75*float(i)/n*2.0*math.pi + deltaT
            x = r*math.sin(t) #+ np.random.uniform(-1,1)*noise
            y = r * math.cos(t)  # + np.random.uniform(-1,1)*noise


            if addSin:
                Xdata.append([x, y,math.sin(x),math.sin(y)])
            else:
                Xdata.append([x, y])

            Ydata.append(label)


    genSpiral(0,1.0)
    genSpiral(math.pi,0.0)

    Xdata = np.array(Xdata)
    Ydata = np.array(Ydata)
    return Xdata,Ydata


"""Code to call the above code""" 

Xdata,Ydata = gen2Spiral(addSin=True)
nn = NeuralNetwork(4,[4,2,1],lr=0.05)
# Xdata,Ydata = spiralgen.gen2Spiral()
# nn = NeuralNetwork(2,[8,8,5,1],lr=0.003)
epoch = 1001

X_train, X_test, y_train, y_test = train_test_split(Xdata, Ydata, test_size=0.30,random_state=1010)

print(X_train.shape,X_test.shape)

nn.train(epoch,X_train,y_train,X_test=X_test,y_test=y_test,
         monitor_train_acc=True,monitor_test_acc=True,monitor_checkpoint=100)<jupyter_output>[4, 2, 1]
((350, 4), (150, 4))
0 Train acc:58.285714 mse:0.269723 lr:0.050000
0 Test acc:50.666667 mse:0.323026 lr:0.050000
100 Train acc:89.428571 mse:0.099129 lr:0.050000
100 Test acc:86.000000 mse:0.123764 lr:0.050000
200 Train acc:94.857143 mse:0.036889 lr:0.050000
200 Test acc:90.666667 mse:0.054077 lr:0.050000
300 Train acc:96.857143 mse:0.025165 lr:0.050000
300 Test acc:92.000000 mse:0.044050 lr:0.050000
400 Train acc:97.428571 mse:0.019562 lr:0.050000
400 Test acc:92.666667 mse:0.039441 lr:0.050000
500 Train acc:98.857143 mse:0.015815 lr:0.050000
500 Test acc:92.666667 mse:0.035983 lr:0.050000
600 Train acc:99.142857 mse:0.013021 lr:0.050000
600 Test acc:92.666667 mse:0.033277 lr:0.050000
700 Train acc:99.428571 mse:0.010950 lr:0.050000
700 Test acc:93.333333 mse:0.031381 lr:0.050000
800 Train acc:99.428571 mse:0.009393 lr:0.050000
800 Test acc:93.333333 mse:0.030038 lr:0.050000
900 Train acc:99.428571 mse:0.008201 lr:0.050000
900 Test acc:93.333333 mse:0.029114 lr:0.050000
100[...]
{'fraction_non_alphanumeric': 0.14654140186055079, 'fraction_numerical': 0.06152712535691259, 'mean_word_length': 2.648521505376344, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '83093', 'license': 'Apache-2.0', 'n_tokens_mistral': 4593, 'n_tokens_neox': 4141, 'n_words': 693, 'path': 'notebooks/Neural_networks/4-neural-networks-part-3-feedforward-neural-network.ipynb', 'repo_name': 'rakesh-malviya/MLCodeGems', 'size': '10857'}