<jupyter_start><jupyter_text>Test All Functions<jupyter_code>data = get_data()
word2Ind, Ind2word = get_dict(data)
tmp_V = len(word2Ind)
tmp_C = 2
tmp_N = 50
tmp_batch_size = 4

tmp_x, tmp_y = next(get_batches(data, word2Ind, tmp_V, tmp_C, tmp_batch_size))
print(tmp_x.shape, tmp_y.shape)


W1, W2, b1, b2 = initialize_model(tmp_N, tmp_V)
tmp_z, tmp_h = forward_prop(tmp_x, W1, W2, b1, b2)
tmp_yhat = softmax(tmp_z)
tmp_cost = compute_cost(tmp_y, tmp_yhat, tmp_batch_size)
print(tmp_cost)<jupyter_output>(5775, 4) (5775, 4)
12.812297426050804<jupyter_text>Backpropagation\begin{align} \frac{\partial J}{\partial \mathbf{W_1}} &= \rm{ReLU}\left ( \mathbf{W_2^\top} (\mathbf{\hat{y}} - \mathbf{y})\right )\mathbf{x}^\top \tag{1}\\ \frac{\partial J}{\partial \mathbf{W_2}} &= (\mathbf{\hat{y}} - \mathbf{y})\mathbf{h^\top} \tag{1}\\ \frac{\partial J}{\partial \mathbf{b_1}} &= \rm{ReLU}\left ( \mathbf{W_2^\top} (\mathbf{\hat{y}} - \mathbf{y})\right ) \tag{3}\\ \frac{\partial J}{\partial \mathbf{b_2}} &= \mathbf{\hat{y}} - \mathbf{y} \tag{4}\end{align}  Ustteki backprop islemi tek bir ifade icin batch icin alinirsa belli degisiklikler olacaktir<jupyter_code>def back_prop(x, yhat, y, h, W1, W2, b1, b2, batch_size):
    '''
    Inputs: 
        x:  average one hot vector for the context 
        yhat: prediction (estimate of y)
        y:  target vector
        h:  hidden vector (see eq. 1)
        W1, W2, b1, b2:  matrices and biases  
        batch_size: batch size 
     Outputs: 
        grad_W1, grad_W2, grad_b1, grad_b2:  gradients of matrices and biases   
    '''
    grad_W2 = (1 / batch_size) * np.dot(yhat - y, h.T)
    grad_b2 = (1 / batch_size) * np.sum(yhat - y, axis = 1, keepdims=True)
    grad_W1 = (1 / batch_size) * np.dot(np.maximum(0, np.dot(W2.T, yhat - y)), x.T)
    grad_b1 = (1 / batch_size) * np.sum(np.maximum(0, np.dot(W2.T, yhat - y)), axis = 1, keepdims=True)
    return grad_W1, grad_W2, grad_b1, grad_b2
def update_parameters(parameters, grads, alpha):
    W1, W2, b1, b2 = parameters
    grad_W1, grad_W2, grad_b1, grad_b2 = grads
    W1 = W1 - alpha * grad_W1
    W2 = W2 - alpha * grad_W2
    b1 = b1 - alpha * grad_b1
    b2 = b2 - alpha * grad_b2
    return W1, W2, b1, b2
def gradient_descent(data, word2Ind, N, V, num_iters, alpha=0.03, batch_size = 128, C = 2):
    
    '''
    This is the gradient_descent function
    
      Inputs: 
        data:      text
        word2Ind:  words to Indices
        N:         dimension of hidden vector  
        V:         dimension of vocabulary 
        num_iters: number of iterations 
        alpha : learning rate
        batch_size : number of examples in each batch
        C : context half size
     Outputs: 
        W1, W2, b1, b2:  updated matrices and biases after num_iters iterations

    '''
    parameters = initialize_model(N, V)
    
    i = 0
    for x, y in get_batches(data, word2Ind, V, C, batch_size):
        z, h = forward_prop(x, *parameters)
        yhat = softmax(z)
        cost = compute_cost(y, yhat, batch_size)
        grads = back_prop(x, yhat, y, h, *parameters, batch_size)
        parameters = update_parameters(parameters, grads, alpha)
        i += 1
        if i == num_iters:
            break
        if (i + 1) % 10 == 0:
            print(f"{i+1}.iter Cost : {cost:.6f}")
        if i % 100 == 0:
            alpha *= 0.66
    return W1, W2, b1, b2
data = get_data()
word2Ind, Ind2word = get_dict(data)
V = len(word2Ind)
N = 50
C = 2
batch_size = 128
alpha = 0.03
num_iters = 150

gradient_descent(data, word2Ind, N, V, num_iters, alpha, batch_size, C)
num_iters = 150
alpha = 0.03
gradient_descent(data, word2Ind, N, V, num_iters, alpha, batch_size, C)
# visualizing the word vectors here
from matplotlib import pyplot
%config InlineBackend.figure_format = 'svg'
words = ['king', 'queen','lord','man', 'woman','dog','wolf',
         'rich','happy','sad']

embs = (W1.T + W2)/2.0 # bu satira dikkat burada embedding vektoru aliniyor.
 
# given a list of words and the embeddings, it returns a matrix with all the embeddings
idx = [word2Ind[word] for word in words]
X = embs[idx, :]
print(X.shape, idx)  # X.shape:  Number of words of dimension N each<jupyter_output>(10, 50) [2744, 3949, 2960, 3022, 5672, 1452, 5671, 4189, 2315, 4276]
{'fraction_non_alphanumeric': 0.1375996249413971, 'fraction_numerical': 0.04992967651195499, 'mean_word_length': 3.01789077212806, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '98125', 'license': 'MIT', 'n_tokens_mistral': 1771, 'n_tokens_neox': 1655, 'n_words': 445, 'path': 'CBOW.ipynb', 'repo_name': 'fturkmencs/pytorch-nlp', 'size': '4266'}