<jupyter_start><jupyter_text>Neural Networks---<jupyter_code>import numpy as np
class Layer:
  def __init__(self):
    self.input = None
    self.output = None

  def forward_propagation(self, input):
    raise NotImplementedError

  def backward_propagation(self, output_error, learning_rate):
    raise NotImplementedError
# Fully connected layer
class FCLayer(Layer):
  def __init__(self, input_size, output_size):
    self.weights = np.random.rand(input_size, output_size) - 0.5
    self.bias = np.random.rand(1, output_size) - 0.5
  
  def forward_propagation(self, input_data):
    self.input = input_data
    self.output = np.dot(self.input, self.weights) + self.bias
    return self.output
  
  def backward_propagation(self, output_error, learning_rate):
    input_error = np.dot(output_error, self.weights.T)
    weights_error = np.dot(self.input.T, output_error)
    
    # update params
    self.weights -= learning_rate * weights_error
    self.bias -= learning_rate * output_error
    return input_error
class ActivationLayer(Layer):
  def __init__(self, activation, activation_prime):
    self.activation = activation
    self.activation_prime = activation_prime

  def forward_propagation(self, input_data):
    self.input = input_data
    self.output = self.activation(self.input)
    return self.output

  def backward_propagation(self, output_error, learning_rate):
    return self.activation_prime(self.input) * output_error
# activation function and derivative
def sigmoid(input):
  return 1 / (1 + np.exp(-input))

def sigmoid_prime(input):
  return np.exp(-input) / (1 + np.exp(-input)**2)

# cost function and derivative of cost function
def mse(y_true, y_pred):
  return np.mean(np.power(y_true - y_pred, 2))

def mse_prime(y_true, y_pred):
  return 2 * (y_pred - y_true) / y_true.size
class Network:
  def __init__(self):
    self.layers = []
    self.loss = None
    self.loss_prime = None

  def add(self, layer):
    self.layers.append(layer)
  
  def use(self, loss, loss_prime):
    self.loss = loss
    self.loss_prime = loss_prime

  def predict(self, input_data):
    samples = len(input_data)
    result = []

    for i in range(samples):
      output = input_data[i]
      
      for layer in self.layers:
        output = layer.forward_propagation(output)
      
      result.append(output)
    
    return output
  
  def fit(self, x_train, y_train, epochs, learning_rate):
    # sample dimension first
    samples = len(x_train)

    # training loop
    for i in range(epochs):
      err = 0
      for j in range(samples):
        # forward propagation
        output = x_train[j]
        for layer in self.layers:
          output = layer.forward_propagation(output)

        # compute loss (for display purpose only)
        err += self.loss(y_train[j], output)

        # backward propagation
        error = self.loss_prime(y_train[j], output)
        for layer in reversed(self.layers):
            error = layer.backward_propagation(error, learning_rate)

      # calculate average error on all samples
      err /= samples
      print('epoch %d/%d   error=%f' % (i+1, epochs, err))
      

from keras.datasets import mnist
from keras.utils import np_utils

(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 1, 28 * 28)
x_train = x_train.astype('float32')
x_train /= 255
y_train = np_utils.to_categorical(y_train)

x_test = x_test.reshape(x_test.shape[0], 1, 28 * 28)
x_test = x_test.astype('float32')
x_test /= 255
y_test = np_utils.to_categorical(y_test)

net = Network()
net.add(FCLayer(28*28, 100))
net.add(ActivationLayer(sigmoid, sigmoid_prime))
net.add(FCLayer(100, 50))
net.add(ActivationLayer(sigmoid, sigmoid_prime))
net.add(FCLayer(50, 10))
net.add(ActivationLayer(sigmoid, sigmoid_prime))

net.use(mse, mse_prime)
net.fit(x_train[0:1000], y_train[0:1000], epochs=35, learning_rate=0.1)

out = net.predict(x_test[0:3])
print("\n")
print("predicted values : ")
print(out, end="\n")
print("true values : ")
print(y_test[0:3])<jupyter_output>epoch 1/35   error=0.091170
epoch 2/35   error=0.076424
epoch 3/35   error=0.063071
epoch 4/35   error=0.052722
epoch 5/35   error=0.044695
epoch 6/35   error=0.038566
epoch 7/35   error=0.033817
epoch 8/35   error=0.029921
epoch 9/35   error=0.026651
epoch 10/35   error=0.023925
epoch 11/35   error=0.021648
epoch 12/35   error=0.019713
epoch 13/35   error=0.018051
epoch 14/35   error=0.016611
epoch 15/35   error=0.015348
epoch 16/35   error=0.014224
epoch 17/35   error=0.013212
epoch 18/35   error=0.012293
epoch 19/35   error=0.011457
epoch 20/35   error=0.010697
epoch 21/35   error=0.010005
epoch 22/35   error=0.009376
epoch 23/35   error=0.008804
epoch 24/35   error=0.008284
epoch 25/35   error=0.007813
epoch 26/35   error=0.007385
epoch 27/35   error=0.006995
epoch 28/35   error=0.006640
epoch 29/35   error=0.006317
epoch 30/35   error=0.006023
epoch 31/35   error=0.005753
epoch 32/35   error=0.005505
epoch 33/35   error=0.005276
epoch 34/35   error=0.005064
epoch 35/35   error=0.0[...]
{'fraction_non_alphanumeric': 0.11403682439120966, 'fraction_numerical': 0.08711146307661849, 'mean_word_length': 3.535008976660682, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '622021', 'license': 'MIT', 'n_tokens_mistral': 2106, 'n_tokens_neox': 1858, 'n_words': 415, 'path': 'Neural_Networks.ipynb', 'repo_name': 'aaronyu888/mat-494-notebooks', 'size': '5051'}