<jupyter_start><jupyter_text>Logistic regression cannot form more complex hypotheses as it is only a linear classifier (You could add more features (such as polynomial features) to logistic regression, but that can be very expensive to train.)<jupyter_code>handwritten_digits = loadmat('ex3data1.mat')
handwritten_digits.keys()
features = handwritten_digits['X']
# X = column_stack((ones((features.shape[0],1)), features))
m, n = features.shape

y = handwritten_digits['y']
features.shape, y.shape
# Loading Saved Neural Network Parameters ...
weight = loadmat('ex3weights.mat')
weight.keys()
def sigmoid(z): 
    return 1/(1+exp(-z))


def predict_from_three_layer_NN(Theta1, Theta2, X):
    m, _ = X.shape
    A_1 = np.c_[ones((m)), X] # (5000, 400)
    
    Z_2 = Theta1.dot(A_1.T) # (25, 401) * (401, 5000)
    A_tmp = sigmoid(Z_2).T # (5000, 25)    
    A_2 = np.c_[(ones((m)), A_tmp)] # (5000, 26) 
    
    Z_3 = Theta2.dot(A_2.T) # (10, 26) * (26, 5000) 
    A_3 = sigmoid(Z_3).T # (5000, 10)
    
    return A_3
print(weight['Theta1'].shape, weight['Theta2'].shape)

W1 = np.random.randn(25, 401) * 0.01
W2 = np.random.randn(10, 26) * 0.01
pred = predict_from_three_layer_NN(W1, W2, features)
print(pred.argmax(axis=1)[:10])
incidence_y = np.zeros((y.size, 10))

incidence_y[np.arange(y.size), y.ravel()-1] = 1  # (5000, 10)
print(incidence_y.shape, pred.shape)
def cross_entropy_loss(y, a):
    return -(y * np.log(a) + (1 - y) * np.log(1 - a))

def cost():
    loss_matrix = cross_entropy_loss(incidence_y, pred)
    sum_over_all_examples = np.sum(loss_matrix, axis=0)/loss_matrix.shape[0]
    print(sum_over_all_examples.shape)
cost()<jupyter_output>(10,)
{'fraction_non_alphanumeric': 0.16316736652669467, 'fraction_numerical': 0.0647870425914817, 'mean_word_length': 4.329073482428115, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '434622', 'license': 'MIT', 'n_tokens_mistral': 740, 'n_tokens_neox': 662, 'n_words': 144, 'path': 'ml/Neural Network - Feedforward.ipynb', 'repo_name': 'disooqi/learning-machine-learning', 'size': '1667'}