<jupyter_start><jupyter_text>Problem 5  Load data, randomize order, then plot a few examples...<jupyter_code>alldata_t = np.loadtxt('./data/digitstrain.txt', delimiter=',')
np.random.seed(42)
np.random.shuffle(alldata_t)
y_t = alldata_t[:,-1].astype(int)
print(y_t.size)
y_t[:100].reshape((10,10))  # Labels...
digits_t = alldata_t[:,:-1]
print(digits_t[0].size)
#nnbp.plot_1(digits_t[0])
nnbp.plot_100(digits_t)<jupyter_output>784<jupyter_text>a) Basic Generalization ```[784] -> [100] -> [10]dim(W1) = [100, 784]dim(W2) = [10, 100]h(0) = X [784] = inputPreAct(1) [100] = W1*h(0) + b1h(1) [100] = sigmoid(PreAct(1))PreAct(2) [10] = W2*h(1) + b2output [10] = softmax(PreAct(2))```<jupyter_code>def sig(x):                                        
    return 1 / (1 + np.exp(-x))

def w_scale(fan_in=100, fan_out=100):
    return 4*np.sqrt(6/(fan_in+fan_out))

def softmax(x):
    """Numerically stable version..."""
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum()

def d_sig(x):
    return sig(x)*(1-sig(x))

def binary_vec_plot(vec, title=""):
    plt.matshow(np.matrix(vec), cmap=mpl.cm.binary)
    plt.title(title)
    
def grad_vec_plot(vec, title=""):
    plt.matshow(np.matrix(vec), cmap=mpl.cm.RdBu)
    plt.title(title)


b1 = np.zeros(100)
b2 = np.zeros(10)
                
W1 = np.random.uniform(w_scale(784, 100), -w_scale(784, 100), size=(100,784))
Wout = np.random.uniform(w_scale(100, 10), -w_scale(100, 10), size=(10,100))
print(digits_t.shape)
digits_t[0].shape<jupyter_output><empty_output><jupyter_text>Forward Pass<jupyter_code>h0 = digits_t[n]
# Hidden Layer 1
PreAct1 = W1 @ h0 + b1
h1 = sig(PreAct1)
# Output Layer
PreActOut = Wout @ h1 + b2
output = softmax(PreActOut)

print("Sum: ",sum(output))
output<jupyter_output><empty_output><jupyter_text>Backpropagation Implementation<jupyter_code>rate = 0.1
# Output of network 
binary_vec_plot(output, "Forward-pass prediction")

# True value
indicator_vector = np.zeros(10)
indicator_vector[y_t[n]] = 1  # Set 
indicator_vector
binary_vec_plot(indicator_vector, "Indicator Vector")

#### Start output layer
# Resulting gradient at lowest level (inverse sign from notes)
PreActOut_grad = (indicator_vector - output)  # [10]
# How much cost will change per pre-activation (at X)
grad_vec_plot(PreActOut_grad, "PreActOut Grad")

# connection weight deltas
b2_grad = PreActOut_grad
Wout_grad = np.array(np.matrix(PreActOut_grad).T @ np.matrix(h1))     # [10,100]
grad_vec_plot(Wout_grad, "Wout Grad")

# Apply updates
b2 = b2 - rate*b2_grad
Wout = Wout - rate*Wout_grad
#### End Output Layer

#### Start hidden layer 1
# How much cost will change per hidden unit activation value
h1_grad = Wout.T @ PreActOut_grad  # [100]
grad_vec_plot(h1_grad, "h1 Grad")

PreAct1_grad = np.multiply(h1_grad,d_sig(PreAct1)) # Element-wise multiplication
grad_vec_plot(PreAct1_grad, "PreAct1 Grad")

b1_grad = PreAct1_grad
W1_grad = np.matrix(PreAct1_grad).T @ np.matrix(h0)
grad_vec_plot(W1_grad, "W1 Grad")

# Apply updates
b1 = b1 - rate*b1_grad
W1 = W1 - rate*W1_grad
#### End hidden layer 1<jupyter_output><empty_output><jupyter_text>Weight matricies<jupyter_code>grad_vec_plot(Wout, "Wout")

grad_vec_plot(W1, "W1")<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>output
indicator_vector
np.log(output)
outputs = []
cross_entropy_errors = [0]
cross_entropy_per_epoch = []

rate = 0.1
b1 = np.zeros(100)
b2 = np.zeros(10)
W1 = np.random.uniform(w_scale(784, 100), -w_scale(784, 100), size=(100,784))
Wout = np.random.uniform(w_scale(100, 10), -w_scale(100, 10), size=(10,100))

for epoch in range(200):
    for n in range(len(digits_t)):
        ###### Forward Pass
        h0 = digits_t[n]
        # Hidden Layer 1
        PreAct1 = W1 @ h0 + b1
        h1 = sig(PreAct1)
        # Output Layer

        PreActOut = Wout @ h1 + b2
        output = softmax(PreActOut)

        # Backward Pass
        # Output of network 

        # True value
        indicator_vector = np.zeros(10)
        indicator_vector[y_t[n]] = 1  # Set  
        outputs.append(indicator_vector)


        cross_entropy_errors.append(np.sum(-np.multiply(np.log(output),indicator_vector)))

        #### Start output layer
        # Resulting gradient at lowest level (inverse sign from notes)
        PreActOut_grad = (indicator_vector - output)  # [10]
        # How much cost will change per pre-activation (at X)

        # connection weight deltas
        b2_grad = PreActOut_grad
        Wout_grad = np.array(np.matrix(PreActOut_grad).T @ np.matrix(h1))     # [10,100]

        # Apply updates

        b2 = b2 + rate*b2_grad
        Wout = Wout + rate*Wout_grad
        #### End Output Layer

        #### Start hidden layer 1
        # How much cost will change per hidden unit activation value
        h1_grad = Wout.T @ PreActOut_grad  # [100]
        PreAct1_grad = np.multiply(h1_grad,d_sig(PreAct1)) # Element-wise multiplication

        b1_grad = PreAct1_grad
        W1_grad = np.matrix(PreAct1_grad).T @ np.matrix(h0)
        # Apply updates
        b1 = b1 + rate*b1_grad
        W1 = W1 + rate*np.array(W1_grad)
        #### End hidden layer 1
    
    cross_entropy_per_epoch.append(np.average(cross_entropy_errors))
    cross_entropy_errors = []
# Output of network 
binary_vec_plot(output, "Forward-pass prediction")
binary_vec_plot(indicator_vector, "Indicator Vector")
grad_vec_plot(PreActOut_grad, "PreActOut Grad")
grad_vec_plot(Wout_grad, "Wout Grad")
grad_vec_plot(Wout, "Wout")
grad_vec_plot(h1_grad, "h1 Grad")
grad_vec_plot(PreAct1_grad, "PreAct1 Grad")
grad_vec_plot(W1_grad, "W1 Grad")
grad_vec_plot(W1, "W1")
nnbp.plot_100(W1)
plt.plot(cross_entropy_per_epoch)
plt.gca().set_yscale('log')
plt.title("Cross Entropy vs Epoch")
plt.savefig('5a_crossentropy_vs_epoch.png')
result = nnbp.train_singlelayer(epochs=10, seed=20)
result['ce_per_epoch']
result['ce_per_epoch']
result['ce_per_epoch']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13944829920460314, 'fraction_numerical': 0.04806227788119817, 'mean_word_length': 4.103626943005182, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '643333', 'license': 'Apache-2.0', 'n_tokens_mistral': 2474, 'n_tokens_neox': 2326, 'n_words': 441, 'path': 'hw1_backprop/backprop_implementation_debug.ipynb', 'repo_name': 'sjobeek/deeplearning_cmu', 'size': '5909'}