<jupyter_start><jupyter_text><jupyter_code>import numpy as np

np.random.seed(0)
# Bias weights are always zero-indexed in each node's array. Bias weight is not
# counted into layer sizes.

input_size = 5

layer_sizes = [5, 10, 3]

nn_list = [[[]] * input_size] # A bit wasteful computationally, but helps keep
                              # the initialization algorithm concise.
for layer_size in layer_sizes:
  # Note that each node has an additional weight initialized for its bias.
  nn_list.append([np.random.random(len(nn_list[-1]) + 1) for
                  node in
                  range(layer_size)])

del nn_list[0]

nn_list
def feed_forward(network, inputs):
  """
  Here we use a typical densely-connected set of layers. I forget the term for
  this.
  """

  bias = 1.0

  def activation(input_):
    # We use ReLU here. I'm too lazy to come up with names that avoid keyword
    # conflicts.
    return 0 if input_ < 0 else input_

  def propagate(weight_matrix, inputs):
    """
    Perform matrix multiplication of weight matrix with inputs. Columns in
    weight_matrix and rows in inputs should be equal. We then run each result
    in the "potentials" vector through an activation function, finishing by
    prepending a constant bias term to the result vector.

    Note that this function assumes a bias has already been added to inputs,
    and is not able to discern whether its output should be the true output
    of the network (you will have to manually prune the bias term from the
    return value with `propagate(...)[1:]` or equivalent on the output layer).

    I know this docstring is kind of long for what this function actually
    accomplishes, but if you're a programmer and you can't be bothered to spend
    a minute reading from time to time then I've got bad news for you.

    Twas bryllyg, and the slythy toves
    Did gyre and gymble in the wabe:
    All mimsy were the borogoves;
    And the mome raths outgrabe.

    Jupyter desperately needs a block collapse feature.
    """

    potentials = np.matmul(weight_matrix, inputs)
    post_activation = [activation(potential) for potential in potentials]
    
    return np.concatenate(([bias], post_activation))

  current_values = np.concatenate(([bias], inputs))

  for layer in network:
    current_values = propagate(layer, current_values)
  
  return current_values[1:] # Remove bias from output.
inputs = np.random.random(input_size)

outputs = feed_forward(nn_list, inputs)

outputs
def error(outputs, ideals):
  """We use root mean square error (RMSE) here."""
  return np.sum((ideals - outputs) ** 2) / outputs.size ** 0.5

ideals = np.array([0.5, 0.75, 0.825]) # I totally made this up.

error(outputs, ideals)
# Now I've just got to implement the actual backprop algorithm, and some kind of
# batching procedure with test data. I'll do that whenever I get bored again.
# It's midnight.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07597535934291581, 'fraction_numerical': 0.009240246406570842, 'mean_word_length': 3.349702380952381, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '286663', 'license': 'MIT', 'n_tokens_mistral': 888, 'n_tokens_neox': 832, 'n_words': 368, 'path': 'Backpropagation.ipynb', 'repo_name': 'hinsley/colabs', 'size': '2922'}