<jupyter_start><jupyter_text>$e_n = - \sum^M_{m=1} y_{n,m} \log(\hat{y}_{n,m})$<jupyter_code>def calc_error(y_hat, y, eps=1e-10):
    """
    Cross-entropy loss between labels, y, and predictions, y_hat 
    """
    error = - np.sum([_y*np.log(_y_hat + eps) for (_y, _y_hat) in zip(y, y_hat)])
    
    return error
print(calc_error([0.2, 0.2, 0.6], [0, 0, 1]))

print(calc_error([0.1, 0.1, 0.8], [0, 0, 1]))

print(calc_error([0.05, 0.05, 0.9], [0, 0, 1]))

print(calc_error([0, 0, 1], [0, 0, 1]))
def softmax(x):
    """Compute softmax values for each sets of scores in x."""
    return np.exp(x) / np.sum(np.exp(x))
softmax([1, 2, 3])
softmax([1000, 2000, 3000])<jupyter_output>/home/ben/miniconda3/envs/pytorch04/lib/python3.6/site-packages/ipykernel_launcher.py:3: RuntimeWarning: overflow encountered in exp
  This is separate from the ipykernel package so we can avoid doing imports until
/home/ben/miniconda3/envs/pytorch04/lib/python3.6/site-packages/ipykernel_launcher.py:3: RuntimeWarning: invalid value encountered in true_divide
  This is separate from the ipykernel package so we can avoid doing imports until<jupyter_text>https://stackoverflow.com/questions/42599498/numercially-stable-softmaxhttps://stackoverflow.com/questions/34968722/how-to-implement-the-softmax-function-in-python<jupyter_code>def softmax(x):
    """Compute softmax values for each sets of scores in x."""
    e_x = np.exp(x - np.max(x))
    return e_x / np.sum(e_x)
softmax([1, 2, 3])
softmax([1000, 2000, 3000])
w = np.random.normal(size=(3,2))
b = np.random.normal(size=(3,1))
def predict(w, b, x):
    pred_vals = np.matmul(w, x)
    pred_vals = softmax(pred_vals)
    pred_class = np.argmax(pred_vals)
    return pred_vals, pred_class
predict(w, b, xs[0])
def predicts(w, b, xs):
    pred_vals, pred_classes = map(np.array, zip(*[predict(w, b, x) for x in xs]))
    return pred_vals, pred_classes
y_hats, c_hats = predicts(w, b, xs)
n_correct = (c_hats == ys).sum()

print(n_correct)
accuracy = n_correct / len(c_hats)

print(accuracy)<jupyter_output>0.335<jupyter_text>https://deepnotes.io/softmax-crossentropyhttps://eli.thegreenplace.net/2016/the-softmax-function-and-its-derivative/http://cs231n.github.io/convolutional-networks/\begin{align*}z &= w_0 x_0 + w_1 x_1 + b\\\hat{y} &= \sigma (z)\\\sigma (z_j) &= \frac{e^{z_j}}{\sum^K_{k=1}e^{z_k}} \\e_n &= - \sum^M_{m=1} y_{n,m} \log(\hat{y}_{n,m}) \\ &= - (y_{n,0} \log(\hat{y}_{n,0}) + y_{n,1} \log(\hat{y}_{n,1}) + y_{n,2} \log(\hat{y}_{n,2}))\end{align*}Derivative of softmax:$$ \sigma (z_j) = \frac{e^{z_j}}{\sum^K_{k=1}e^{z_k}} $$$$\frac{d\sigma (z_j)}{d z_j} = \frac{d}{d z_j} \frac{e^{z_j}}{\sum^K_{k=1}e^{z_k}} $$When $i = j$:$$ \frac{d}{d z_j} \sigma (z_j) = \frac{e^{z_j}}{\sum^K_{k=1}e^{z_k}} = \frac{e^{z_i} \sum^K_{k=1} e^{z_k} - e^{z_j}e^{z_i}}{(\sum^K_{k=1} e^z_k)^2} $$ $$=\frac{e^{z_i}(\sum^K_{k=1}e^{z_k}-e^{z_j})}{(\sum^K_{k=1}e^{z_k})^2}$$$$=\frac{e^{z_j}}{\sum^K_{k=1}e^{z_k}} \frac{(\sum^K_{k=1}e^{z_k}-e^{z_j})}{\sum^K_{k=1}e^{z_k}}$$$$=p_i(1-p_j)$$When $i \neq j$:$$ \frac{d}{d z_j} \sigma (z_j) = \frac{e^{z_j}}{\sum^K_{k=1}e^{z_k}} = \frac{0 - e^{z_j}e^{z_i}}{(\sum^K_{k=1} e^z_k)^2} $$ $$=\frac{-e^{z_j}}{\sum^K_{k=1}e^{z_k}} \frac{(\sum^K_{k=1}e^{z_i})}{\sum^K_{k=1}e^{z_k}}$$$$=-p_j*p_i$$For loss function:$$L= - \sum_iy_i\log(p_i)$$$$=\frac{d L}{\hat{y}_i} = -\sum^ky_k \frac{d \log (p_k)}{d \hat{y}_i}$$$$=-\sum_ky_k\frac{d \log(p_k)}{d p_k} \times \frac{d p_k}{\hat{y_i}}$$$$=- \sum y_k \frac{1}{p_k} \times \frac{d p_k}{d o_i}$$From the softmax derivation earlier:$$\frac{d L}{d o_i} = -y_i(1-p_i)-\sum_{k \neq i} y_k \frac{1}{p_k}(-p_kp_i)$$$$<jupyter_code>def gradient_step(w, b, xs, ys, eta=0.1):
    """
    Performs one gradient descent step using all of the data points
    pred_w0 (float): predicted w0 value
    pred_w1 (float): predicted w1 value
    pred_b (float): predicted b value
    xs (ndarray[float,float]): x values for data points
    ys (ndarray[float]): y values for data points
    eta (float): learning rate
    """
    
    N = len(xs) #number of examples, used to average gradients
    grads_w0 = [0, 0, 0]
    grads_w1 = [0, 0, 0]
    grads_b = [0, 0, 0]
    
    for x, y in zip(xs, ys):
        y_hat, _ = predict(w, b, x) #get prediction
        
        #de = ? #derivative of error w.r.t. prediction
        #dz = ? #derivative of softmax
        
        
        #de = (y_hat - y) / (y_hat * (1 - y_hat)) #derivative of error w.r.t. prediction 
        #dz = y_hat * (1 - y_hat) #derivative of sigmoid
        dedz = y_hat - y #simplified equation for the two derivatives above
        dw0 = x[0] #derivative of sigmoid w.r.t w0
        dw1 = x[1] #derivative of sigmoid w.r.t w1
        grad_w0 += (1/N) * dedz * dw0 #summing averaged gradients for w0
        grad_w1 += (1/N) * dedz * dw1 #summing averaged gradients for w1
        grad_b += (1/N) * dedz #summing averaged gradients for b
        
    new_w0 = pred_w0 - (eta * grad_w0) #parameter update for w0
    new_w1 = pred_w1 - (eta * grad_w1) #parameter update for w1
    new_b = pred_b - (eta * grad_b) #parameter update for b
    
    return new_w0, new_w1, new_b
gradient_step(w, b, xs, ys)<jupyter_output>[4.13837282e-08 1.67298610e-04 9.99832660e-01]
{'fraction_non_alphanumeric': 0.2091065620814999, 'fraction_numerical': 0.043045724124736943, 'mean_word_length': 3.827331486611265, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '523385', 'license': 'MIT', 'n_tokens_mistral': 2440, 'n_tokens_neox': 2301, 'n_words': 469, 'path': '4 - Multinomial Logistic Regression.ipynb', 'repo_name': 'bentrevett/gradient-descent', 'size': '5227'}