<jupyter_start><jupyter_text>Forward pass<jupyter_code># create a single input example (lower-case x)
x = np.random.random((1,1))
x, x.shape
np.random.seed(1)
# normalizing a value

n_examples = 100000
d0 = np.random.random((n_examples,1))  # before norm
d1 = 2*np.random.random((n_examples,1))  # after scaling (now between 0 and 2.0)
d2 = 2*np.random.random((n_examples,1))-1  # after shifting - final norm result

print("min, mean, max")
print([np.around(d, decimals=3) for d in [d0.min(), d0.mean(), d0.max()]])
print([np.around(d, decimals=3) for d in [d1.min(), d1.mean(), d1.max()]])
print([np.around(d, decimals=3) for d in [d2.min(), d2.mean(), d2.max()]])

np.random.seed(1)
# create normalized input - single example with single node
x = 2*np.random.random((1,1))-1
x, x.shape
# create single node weight
w = 2*np.random.random((1,1))-1
w, w.shape
# create single node bias
b = 2*np.random.random((1,1))-1
b, b.shape
# forward (linear) pass
y = x*w +b
y, y.shape
# define activation function
sigmoid = lambda x: 1/(1+np.exp(-x))
# forward (non-linear) pass
y = sigmoid(x*w +b)
y, y.shape
# when we will use vectors and matrices
# the mul (*) operation is replaced with dot product
y = sigmoid(x.dot(w) +b)
y, y.shape
'''
example with multiple input nodes (features) output nodes (classes/...)
still using a single example
'''
n_input = 3
n_output = 2

n_examples = 1
np.random.seed(1)
x = 2 * np.random.random((n_examples, n_input)) - 1
x, x.shape
# weights - now use upper-case W because is a matrix
# weights shape should match the input and output of the x*W result
W = 2 * np.random.random((n_input, n_output)) - 1
W, W.shape
# bias is with the shape of a single output example
b = 2 * np.random.random((1, n_output)) - 1
b, b.shape
# forward pass
y = sigmoid(x.dot(W) +b)
y, y.shape
# multiple examples
n_input = 3
n_output = 2

n_examples = 5
np.random.seed(1)
# inputs are now in matrix, so use upper-case X
# each row is an example
X = 2 * np.random.random((n_examples, n_input)) - 1
X, X.shape
# weights shape does not depend on the number of examples
# only on the sape of a single input and output example
W = 2 * np.random.random((n_input, n_output)) - 1
W, W.shape
np.random.seed(1)
# bias is with the shape of a single output example - still lower-case b because is a vector
b = 2 * np.random.random((1, n_output)) - 1
b, b.shape
# forward pass
# the same code of single example - dot product "magic"
# the dot product is applied to each x example in X

# output is also a matrix - so now use upper-case Y

y0 = sigmoid(X[0].dot(W) +b)
Y = sigmoid(X.dot(W) +b)
print(y0, y0.shape)
print(Y, Y.shape)
# Adding a hidden layer
n_input = 3
n_hidden = 4
n_output = 2

n_examples = 5
np.random.seed(1)
X = 2 * np.random.random((n_examples, n_input)) - 1
X, X.shape

# hidden layer shape depend on its number of nodes
# AND on the number of examples

H = 2 * np.random.random((n_examples, n_input)) - 1
H, H.shape
# now there are multiple weight matrices and bias vectors

# input to hidden
Wih = 2 * np.random.random((n_input, n_hidden)) - 1
bih = 2 * np.random.random((1, n_hidden)) - 1

# hidden to output
Who = 2 * np.random.random((n_hidden, n_output)) - 1
bho = 2 * np.random.random((1, n_output)) - 1
# forward pass through all layers
H = sigmoid(X.dot(Wih) + bih)
Y = sigmoid(H.dot(Who) + bho)

Y, Y.shape<jupyter_output><empty_output><jupyter_text>Backpropagation<jupyter_code>sigmoid = lambda x: 1/(1+np.exp(-x))
sigmoid_backward = lambda x: sigmoid(x) * (1-sigmoid(x))
np.random.seed(1)
n_input = 3
n_hidden = 4
n_output = 2

n_examples = 5
# for simplicity we won't use biases this time

X = np.random.random((n_examples, n_input)) > 0.5
H = 2 * np.random.random((n_examples, n_input)) - 1
Y = np.zeros((n_examples, n_output))  # the actual outputs

# input to hidden
Wih = 2 * np.random.random((n_input, n_hidden)) - 1

# hidden to output
Who = 2 * np.random.random((n_hidden, n_output)) - 1
# forward pass
H = sigmoid(X.dot(Wih))
Yp = sigmoid(H.dot(Who))  # output predictions
error = Y - Yp  # per example<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14433243176788788, 'fraction_numerical': 0.027046963363658718, 'mean_word_length': 4.04089219330855, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '310056', 'license': 'MIT', 'n_tokens_mistral': 1629, 'n_tokens_neox': 1548, 'n_words': 502, 'path': 'my_toy_examples.ipynb', 'repo_name': 'AVTsoof/deep-learning-v2-pytorch', 'size': '4067'}