<jupyter_start><jupyter_text>Section 1<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
class NeuralNetwork(object):
    def __init__(self, layers = [1 , 10, 1], activations=['sigmoid', 'sigmoid']):
        
        self.layers = layers
        self.activations = activations
        self.weights = []
        self.biases = []
        for i in range(len(layers)-1):
            self.weights.append(np.random.randn(layers[i+1], layers[i]))
            self.biases.append(np.random.randn(layers[i+1], 1))
    
    def fit(self, x):
        
        a = np.copy(x)
        z_s = []
        a_s = [a]
        for i in range(len(self.weights)):
            activation_function = self.getActivationFunction(self.activations[i])
            z_s.append(self.weights[i].dot(a) + self.biases[i])
            a = activation_function(z_s[-1])
            a_s.append(a)
        return (z_s, a_s)
    
    def backpropagation(self,y, z_s, a_s):

            dw = []  
            db = []  
            errors = [None] * len(self.weights)  
            
            errors[-1] = ((y-a_s[-1])*(self.getDerivitiveActivationFunction(self.activations[-1]))(z_s[-1]))
            
            for i in reversed(range(len(errors)-1)):
                errors[i] = self.weights[i+1].T.dot(errors[i+1])*(self.getDerivitiveActivationFunction(self.activations[i])(z_s[i]))        
            
            batch_size = y.shape[1]
            db = [d.dot(np.ones((batch_size,1)))/float(batch_size) for d in errors]
            dw = [d.dot(a_s[i].T)/float(batch_size) for i,d in enumerate(errors)]
            
            return dw, db
            
    def train(self, x, y, batch_size=512, epochs=1000, lr = 0.1):
    
        for e in range(epochs): 
            i=0
            while(i<len(y)):
                x_batch = x[i:i+batch_size]
                y_batch = y[i:i+batch_size]
                i = i+batch_size
                z_s, a_s = self.fit(x_batch)
                dw, db = self.backpropagation(y_batch, z_s, a_s)
                self.weights = [w+lr*dweight for w,dweight in  zip(self.weights, dw)]
                self.biases = [w+lr*dbias for w,dbias in  zip(self.biases, db)]
                if e == epochs - 1 : print("loss = {}".format(np.linalg.norm(a_s[-1]-y_batch) ))
    
    @staticmethod
    def getActivationFunction(name):
        return lambda x : np.exp(x)/(1+np.exp(x))
        
    
    @staticmethod
    def getDerivitiveActivationFunction(name):
        sig = lambda x : np.exp(x)/(1+np.exp(x))
        return lambda x :sig(x)*(1-sig(x)) 

      
def main(layer , activations , X, Y, X_test, Y_test, epochs = 50, lr = 0.5, batch_size = 512):
    
    nn = NeuralNetwork(layer,activations)
    
    nn.train(X, Y, epochs, batch_size, lr )
    
    _, train = nn.fit(X)
    
    plt.scatter(X.flatten(), Y.flatten())
    plt.scatter(X.flatten(), train[-1].flatten())
    plt.show()

    _, prediction = nn.fit(X_test)

    plt.plot(X_test.flatten(), Y_test.flatten(), label='Actual')
    plt.plot(X_test.flatten() ,prediction[-1].flatten(), label='Predicted')
    plt.legend()
    plt.show()<jupyter_output><empty_output><jupyter_text>Section 2 Sin(x)<jupyter_code>x = 2*np.pi*np.random.rand(1000).reshape(1, -1)
y = np.sin(x)

x_test = np.arange(-2*np.pi, 2*np.pi, 0.1).reshape(1, -1)
y_test = np.sin(x_test)

layer = [1, 10, 1]
activations =['sigmoid', 'sigmoid']
main(layer , activations, x, y, x_test, y_test, 1500, 0.5)
x = 2*np.pi*np.random.rand(1000).reshape(1, -1)
y = np.sin(x)

x_test = np.arange(0, 10*np.pi, 0.1).reshape(1, -1)
y_test = np.sin(x_test)

layer = [1, 100, 1]
activations =['sigmoid', 'sigmoid']
main(layer , activations, x, y, x_test, y_test, 1500, 0.5)
x = 2*np.pi*np.random.rand(5000).reshape(1, -1)
y = np.sin(x)

x_test = np.arange(-2*np.pi, 2*np.pi, 0.1).reshape(1, -1)
y_test = np.sin(x_test)

layer = [1, 128, 64, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid']
main(layer , activations, x, y, x_test, y_test, 1000, 0.6)
x = 4*np.pi*np.random.rand(4000).reshape(1, -1)
y = np.sin(x)

x_test = np.arange(-10*np.pi, 10*np.pi, 0.1).reshape(1, -1)
y_test = np.sin(x_test)

layer = [1, 128, 64, 32, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']
main(layer , activations, x, y, x_test, y_test, 5000, 0.6)<jupyter_output><empty_output><jupyter_text>Same as Keras<jupyter_code>x = 4*np.pi*np.random.rand(10000).reshape(1, -1)
y = np.sin(x)

x_test = np.arange(-10*np.pi, 10*np.pi, 0.1).reshape(1, -1)
y_test = np.sin(x_test)

layer = [1, 128, 64, 16, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']
main(layer , activations, x, y, x_test, y_test, 1000, 0.6)<jupyter_output><empty_output><jupyter_text>X^2 same as keras<jupyter_code>x = np.random.uniform(-1, 1, 2000).reshape(1, -1)
y = x**2

x_test = np.arange(-2, 2, 0.1).reshape(1, -1)
y_test = x_test**2

layer = [1, 128, 64, 16, 1]

activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']

main(layer , activations, x, y, x_test, y_test, 1000, 0.6)
x = np.random.uniform(-1, 1, 2000).reshape(1, -1)
y = x**2

x_test = np.arange(-2, 2, 0.1).reshape(1, -1)
y_test = x_test**2

layer = [1, 512, 512, 512, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']

main(layer , activations, x, y, x_test, y_test, 10000, 0.5)<jupyter_output>loss = 5.986933198279538<jupyter_text>sin(x) + x<jupyter_code>x = np.random.uniform(-1, 1, 5000).reshape(1, -1)
y = x + np.sin(x)

x_test = np.arange(-2, 2, 0.1).reshape(1, -1)
y_test = x_test + np.sin(x_test)

layer = [1, 128, 64, 32, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']

main(layer , activations, x, y, x_test, y_test, 2000, 0.5)
x = np.random.uniform(-5, 5, 5000).reshape(1, -1)
y = x + np.sin(x)

x_test = np.arange(-2, 2, 0.1).reshape(1, -1)
y_test = x_test + np.sin(x_test)

layer = [1, 128, 64, 32, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']

main(layer , activations, x, y, x_test, y_test, 2000, 0.5)
x = np.random.uniform(-1, 1, 8000).reshape(1, -1)
y = x + np.sin(x)

x_test = np.arange(-2, 2, 0.1).reshape(1, -1)
y_test = x_test + np.sin(x_test)

layer = [1, 128, 64, 32, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']

main(layer , activations, x, y, x_test, y_test, 10000, 0.7)<jupyter_output>loss = 72.83148783127781<jupyter_text>Section 3 noisy data sin(x) --noisy<jupyter_code>x = 2*np.pi*np.random.rand(1000).reshape(1, -1)
y = np.sin(x) + np.random.uniform(low=-0.1, high=0.1, size=(len(x)))

x_test = np.arange(0, 2*np.pi, 0.1).reshape(1, -1)
y_test = np.sin(x_test)

layer = [1, 128, 64, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid']
main(layer , activations, x, y, x_test, y_test, 1000, 0.6)
x = 2*np.pi*np.random.rand(1000).reshape(1, -1)
y = np.sin(x) + np.random.uniform(low=-0.7, high=0.7, size=(len(x)))

x_test = np.arange(0, 2*np.pi, 0.1).reshape(1, -1)
y_test = np.sin(x_test)

layer = [1, 128, 64, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid']
main(layer , activations, x, y, x_test, y_test, 1000, 0.6)
x = 2*np.pi*np.random.rand(1000).reshape(1, -1)
y = np.sin(x) + np.random.uniform(low=-9, high=9, size=(len(x)))

x_test = np.arange(0, 2*np.pi, 0.1).reshape(1, -1)
y_test = np.sin(x_test)

layer = [1, 128, 64, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid']
main(layer , activations, x, y, x_test, y_test, 1000, 0.6)<jupyter_output>loss = 305.12507651308323<jupyter_text>X^2 -- noisy<jupyter_code>x = np.random.uniform(-1, 1, 2000).reshape(1, -1)
y = x**2 + np.random.uniform(low=-0.1, high=0.1, size=(len(x)))

x_test = np.arange(-2, 2, 0.1).reshape(1, -1)
y_test = x_test**2

layer = [1, 128, 64, 16, 1]

activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']

main(layer , activations, x, y, x_test, y_test, 1000, 0.6)
x = np.random.uniform(-1, 1, 2000).reshape(1, -1)
y = x**2 + np.random.uniform(low=0.5, high=0.5, size=(len(x)))

x_test = np.arange(-2, 2, 0.1).reshape(1, -1)
y_test = x_test**2

layer = [1, 128, 64, 16, 1]

activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']

main(layer , activations, x, y, x_test, y_test, 1000, 0.6)
x = np.random.uniform(-1, 1, 2000).reshape(1, -1)
y = x**2 + np.random.uniform(low=0.9, high=0.9, size=(len(x)))

x_test = np.arange(-2, 2, 0.1).reshape(1, -1)
y_test = x_test**2

layer = [1, 128, 64, 16, 1]

activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']

main(layer , activations, x, y, x_test, y_test, 1000, 0.6)<jupyter_output>loss = 16.561213387116876<jupyter_text>sin(x) + x --noisy<jupyter_code>x = np.random.uniform(-1, 1, 2000).reshape(1, -1)
y = x + np.sin(x) + np.random.uniform(low=0.9, high=0.9, size=(len(x)))

x_test = np.arange(-2, 2, 0.1).reshape(1, -1)
y_test = x_test + np.sin(x_test)

layer = [1, 512, 255, 128, 1]
activations =['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']

main(layer , activations, x, y, x_test, y_test, 10000, 0.5)<jupyter_output>loss = 34.99607615379013
{'fraction_non_alphanumeric': 0.19845944721341188, 'fraction_numerical': 0.0731762573629361, 'mean_word_length': 3.334315169366716, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '511231', 'license': 'MIT', 'n_tokens_mistral': 4311, 'n_tokens_neox': 3877, 'n_words': 764, 'path': 'MLP_with_Numpy.ipynb', 'repo_name': 'ghazaleh-mahmoodi/-Preceptron-Implementation-without-framework', 'size': '8828'}