<jupyter_start><jupyter_text>Birthing a Simple PerceptronDate May 21, 2018The following experiment implements a perceptron in a Python class with a number of class variables and two class methods: train and predict. A perceptron is the basic atomic working unit of neural networks, popular structures that today helps facilitate computing implementation algorithms for problems in deep learning. More on perceptron --> https://en.wikipedia.org/wiki/PerceptronMore on activation functions -> https://towardsdatascience.com/activation-functions-and-its-types-which-is-better-a9a5310cc8fBlueberryTron named after our dog 'Blueberry'<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
# types of activation functions

def activat1(x):
    return np.tanh(x)         # tanh or f(x) = 1 — exp(-2x) / 1 + exp(-2x);

def activat2(x):              # stochastically determined +/- binary activation
    if x >= 0:
        return 1
    else:
        return -1

def activat3(x):             # sigmoid -> f(x) = 1 / 1 + exp(-x)
    return 1 / (1 + np.exp(-x))

#perceptron class using activate function 2
class perceptron2(object):
    def __init__(self,dim):
        self.dim = dim       #input space dimension
        self.learner = 0.1   #learning rate
        self.w = np.array([0.001]*(dim+1)) #+1 for bias
        self.cycles = 0      #counter for number of loop cycles
        self.converger = 100 #check for convergence
        self.maxiter = 10000 #max num of iterations before the loop stops
        self.toler = 0.001   #tolerance
        self.errors = 0      #number of wrong predictions in the testing set 
    
    #training algorithm
    def train(self, X, y):
        
        little_w = 0                               #counter for little changes of bias
        
        while(little_w < self.converger and self.cycles < self.maxiter):
            
            i = np.random.randint(0, len(X))
            x_vec  = np.concatenate(([1],X[i]))    #random biased input vector
            d_output = y[i]                        #desired output for the selected example
            a_output = np.dot(self.w, x_vec)       #computed actual output 
            
            old_w = self.w
            act = activat2(a_output)               #apply activation to de-linearize actual output 
            self.w += self.learner*(d_output - act)*x_vec #Learning Rule = compare desired output and actual output

            if np.linalg.norm(self.w - old_w) < self.toler:
                little_w += 1
            else:
                little_w = 0                       #if w changed enough, reset the counter

            self.cycles += 1
    
    #predict algorithm        
    def predict(self, X_pred, y_des):
        y_pred = ['error']*len(X_pred)                  #initialize the vector of predictions
        for i in range(len(X_pred)):
            x_vec = np.concatenate(([1],X_pred[i]))     #biased input vector 
            p_output = np.dot(self.w, x_vec)            #compute predicted output
            y_pred[i] = activat2(p_output)              #apply activation to de-linearize computed prediction
            if (y_pred[i] != y_des[i]):                 #if the prediction is incorrect, increment errors counter
                self.errors +=  1       
            
        return y_pred
# Train Set
Npoints = 100
xcomp = np.linspace(2,np.pi, Npoints)

redRdm = np.random.normal(size=Npoints, loc=0, scale=2) # random sample from a normal distribution
redPts = np.array([np.array([xcomp[i],3*xcomp[i] + redRdm[i]]) for i in range(100)])
redDes = np.repeat(-1,Npoints)

blueRdm = np.random.normal(size=Npoints, loc=8, scale=2)
bluePts = np.array([ np.array([xcomp[i],3*xcomp[i] + blueRdm[i]]) for i in range(100)])
blueDes = np.repeat(1,Npoints)

X_train = np.concatenate((redPts,bluePts))
y_train = np.concatenate((redDes,blueDes))
# Test Set
Npoints = 50
xcomp = np.linspace(0, np.pi, Npoints)

redRdm = np.random.normal(size=Npoints, loc=0, scale=2)
redPts = np.array([np.array([xcomp[i],3*xcomp[i] + redRdm[i]]) for i in range(Npoints)])
redDes = np.repeat(-1,Npoints)

blueRdm = np.random.normal(size=Npoints, loc=8, scale=2)
bluePts = np.array([ np.array([xcomp[i],3*xcomp[i] + blueRdm[i]]) for i in range(Npoints)])
blueDes = np.repeat(1,Npoints)

X_test = np.concatenate((redPts,bluePts))
y_test = np.concatenate((redDes,blueDes))
# Model using activate function 2
dimension = X_train.shape[1]
Blueberrytron = perceptron2(dimension)
Blueberrytron.train(X_train, y_train)
Blueberrytron.predict(X_test, y_test)
plt.figure(figsize=(15,10))
for i in range(Npoints):
    plt.scatter(redPts[i,0],redPts[i,1],alpha = 0.3,c='r')
    plt.scatter(bluePts[i,0],bluePts[i,1],alpha = 0.3,c='b')
weights = Blueberrytron.w
plt.plot(xcomp, -(weights[1]*xcomp + weights[0])/weights[2] , 'g-',label='Learned Classifier Line') 
plt.savefig('Blueberrytron.png')
plt.title('Perceptron')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Try TanH<jupyter_code>#perceptron class using activate function 1 tanh
class perceptron1(object):
    def __init__(self,dim):
        self.dim = dim       #input space dimension
        self.learner = 0.1   #learning rate
        self.w = np.array([0.001]*(dim+1)) #+1 for bias
        self.cycles = 0      #counter for number of loop cycles
        self.converger = 100 #check for convergence
        self.maxiter = 10000 #max num of iterations before the loop stops
        self.toler = 0.001   #tolerance
        self.errors = 0      #number of wrong predictions in the testing set 
    
    #training algorithm
    def train(self, X, y):
        
        little_w = 0                               #counter for little changes of bias
        
        while(little_w < self.converger and self.cycles < self.maxiter):
            
            i = np.random.randint(0, len(X))
            x_vec  = np.concatenate(([1],X[i]))    #random biased input vector
            d_output = y[i]                        #desired output for the selected example
            a_output = np.dot(self.w, x_vec)       #computed actual output 
            
            old_w = self.w
            act = activat1(a_output)               #apply activation to de-linearize actual output 
            self.w += self.learner*(d_output - act)*x_vec #Learning Rule = compare desired output and actual output

            if np.linalg.norm(self.w - old_w) < self.toler:
                little_w += 1
            else:
                little_w = 0                       #if w changed enough, reset the counter

            self.cycles += 1
    
    #predict algorithm        
    def predict(self, X_pred, y_des):
        y_pred = ['error']*len(X_pred)                  #initialize the vector of predictions
        for i in range(len(X_pred)):
            x_vec = np.concatenate(([1],X_pred[i]))     #biased input vector 
            p_output = np.dot(self.w, x_vec)            #compute predicted output
            y_pred[i] = activat1(p_output)              #apply activation to de-linearize computed prediction
            if (y_pred[i] != y_des[i]):                 #if the prediction is incorrect, increment errors counter
                self.errors +=  1       
            
        return y_pred
dimension = X_train.shape[1]
Blueberrytron = perceptron1(dimension)
Blueberrytron.train(X_train, y_train)
Blueberrytron.predict(X_test, y_test)
plt.figure(figsize=(15,10))
for i in range(Npoints):
    plt.scatter(redPts[i,0],redPts[i,1],alpha = 0.3,c='r')
    plt.scatter(bluePts[i,0],bluePts[i,1],alpha = 0.3,c='b')
weights = Blueberrytron.w
plt.plot(xcomp, -(weights[1]*xcomp + weights[0])/weights[2] , 'g-',label='Learned Classifier Line') 
plt.savefig('Blueberrytron.png')
plt.title('Perceptron')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Try Sigmoid<jupyter_code>#perceptron class using activate function 3 sigmoid
class perceptron3(object):
    def __init__(self,dim):
        self.dim = dim       #input space dimension
        self.learner = 0.1   #learning rate
        self.w = np.array([0.001]*(dim+1)) #+1 for bias
        self.cycles = 0      #counter for number of loop cycles
        self.converger = 100 #check for convergence
        self.maxiter = 100000 #max num of iterations before the loop stops
        self.toler = 0.001   #tolerance
        self.errors = 0      #number of wrong predictions in the testing set 
    
    #training algorithm
    def train(self, X, y):
        
        little_w = 0                               #counter for little changes of bias
        
        while(little_w < self.converger and self.cycles < self.maxiter):
            
            i = np.random.randint(0, len(X))
            x_vec  = np.concatenate(([1],X[i]))    #random biased input vector
            d_output = y[i]                        #desired output for the selected example
            a_output = np.dot(self.w, x_vec)       #computed actual output 
            
            old_w = self.w
            act = activat3(a_output)               #apply activation to de-linearize actual output 
            self.w += self.learner*(d_output - act)*x_vec #Learning Rule = compare desired output and actual output

            if np.linalg.norm(self.w - old_w) < self.toler:
                little_w += 1
            else:
                little_w = 0                       #if w changed enough, reset the counter

            self.cycles += 1
    
    #predict algorithm        
    def predict(self, X_pred, y_des):
        y_pred = ['error']*len(X_pred)                  #initialize the vector of predictions
        for i in range(len(X_pred)):
            x_vec = np.concatenate(([1],X_pred[i]))     #biased input vector 
            p_output = np.dot(self.w, x_vec)            #compute predicted output
            y_pred[i] = activat3(p_output)              #apply activation to de-linearize computed prediction
            if (y_pred[i] != y_des[i]):                 #if the prediction is incorrect, increment errors counter
                self.errors +=  1       
            
        return y_pred
dimension = X_train.shape[1]
Blueberrytron = perceptron3(dimension)
Blueberrytron.train(X_train, y_train)
Blueberrytron.predict(X_test, y_test)
np.exp(.08310169374426e-268)
plt.figure(figsize=(15,10))
for i in range(Npoints):
    plt.scatter(redPts[i,0],redPts[i,1],alpha = 0.3,c='r')
    plt.scatter(bluePts[i,0],bluePts[i,1],alpha = 0.3,c='b')
weights = Blueberrytron.w
plt.plot(xcomp, -(weights[1]*xcomp + weights[0])/weights[2] , 'g-',label='Learned Line') 
plt.savefig('Blueberrytron.png')
plt.title('Perceptron')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11135084427767354, 'fraction_numerical': 0.021294559099437147, 'mean_word_length': 2.344102885821832, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '457727', 'license': 'MIT', 'n_tokens_mistral': 3587, 'n_tokens_neox': 3342, 'n_words': 868, 'path': 'Exercises/DL/Perceptron/.ipynb_checkpoints/NN - Simple Perceptron-checkpoint.ipynb', 'repo_name': 'ginobaltazar7/Data-Science', 'size': '10660'}