<jupyter_start><jupyter_text>1. Simple Neural NetworkHere we will build a simple neutral network, with a linear layer, a ReLU layer, and a linear layer.- $h = w_1 x$- $h_{ReLU} = ReLU(h)$- $\hat{y}=w_2h_{ReLU}$<jupyter_code>import random

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

USE_CUDA = torch.cuda.is_available()
random.seed(1)
np.random.seed(1)
torch.manual_seed(1)
if USE_CUDA:
    torch.cuda.manual_seed(1)
N = 64
D_IN = 1000
H = 100
D_OUT = 10
NUM_EPOCHS = 500
learning_rate = 1e-6<jupyter_output><empty_output><jupyter_text>1.1 Pure numpy<jupyter_code>x = np.random.randn(N, D_IN)
y = np.random.randn(N, D_OUT)
w1 = np.random.randn(D_IN, H)
w2 = np.random.randn(H, D_OUT)

for epoch in range(NUM_EPOCHS):
    # fowward pass
    h = x.dot(w1)  # (64, 1000)*(1000, 100) = (64, 100)
    h_relu = np.maximum(h, 0)  # (64, 100)
    y_pred = h_relu.dot(w2)  # (64, 100)*(100, 10) = (64, 10)

    loss = np.square(y_pred - y).sum()
    print(f"{epoch}: loss = {loss}")

    # calculate gradients

    # d loss / d y_pred = 2 * (y_pred - y)
    # d loss / d w2 = (d loss / d y_pred) * (d y_pred / d w2)
    y_pred_grad = 2 * (y_pred - y)  # (64,10)
    w2_grad = h_relu.T.dot(y_pred_grad)  # (100, 64)*(64, 10) = (100, 10)

    # d loss / d h_relu =  2 * (y_pred - y) * w2
    h_relu_grad = y_pred_grad.dot(w2.T)  # (64, 10)*(10, 100) = (64, 100)
    h_grad = h_relu_grad.copy()
    h_grad[h < 0] = 0  # (64, 100)
    # d loss / d h =  2 * (y_pred - y) * w2
    w1_grad = x.T.dot(h_grad)  # (1000, 64) * (64, 100) = (1000, 100)

    # update parameters
    w1 -= learning_rate * w1_grad
    w2 -= learning_rate * w2_grad<jupyter_output>0: loss = 24111449.222389005
1: loss = 19076708.530630156
2: loss = 19279786.257235497
3: loss = 22619014.78600002
4: loss = 27088372.012035266
5: loss = 29558662.253394634
6: loss = 26998688.822538935
7: loss = 19671584.37985573
8: loss = 11568014.60135452
9: loss = 5921807.165661035
10: loss = 2970763.928990799
11: loss = 1624117.0378756088
12: loss = 1021610.7409818026
13: loss = 733110.6716428237
14: loss = 576218.4336398772
15: loss = 477345.51577891305
16: loss = 406653.67393033847
17: loss = 351711.40464787965
18: loss = 306912.43438767747
19: loss = 269434.72992538085
20: loss = 237626.80022843674
21: loss = 210365.65284621183
22: loss = 186984.28777858437
23: loss = 166698.63525727473
24: loss = 149040.16256958485
25: loss = 133592.6333383929
26: loss = 120051.31101215395
27: loss = 108141.86333374918
28: loss = 97631.94888989479
29: loss = 88346.45534677307
30: loss = 80101.33248235229
31: loss = 72766.16415836931
32: loss = 66223.31366757283
33: loss = 60371.096216963386
34:[...]<jupyter_text>1.2 Pytorch Tensor<jupyter_code>x = torch.randn(N, D_IN)
y = torch.randn(N, D_OUT)
w1 = torch.randn(D_IN, H)
w2 = torch.randn(H, D_OUT)

for epoch in range(NUM_EPOCHS):
    # fowward pass
    h = x @ w1  # (64, 1000)*(1000, 100) = (64, 100)
    h_relu = F.relu(h)  # (64, 100)
    y_pred = h_relu @ w2  # (64, 100)*(100, 10) = (64, 10)

    loss = (y_pred - y).pow(2).sum().item()
    print(f"{epoch}: loss = {loss}")

    # calculate gradients

    # d loss / d y_pred = 2 * (y_pred - y)
    # d loss / d w2 = (d loss / d y_pred) * (d y_pred / d w2)
    y_pred_grad = 2 * (y_pred - y)  # (64,10)
    w2_grad = h_relu.T @ (y_pred_grad)  # (100, 64)*(64, 10) = (100, 10)

    # d loss / d h_relu =  2 * (y_pred - y) * w2
    h_relu_grad = y_pred_grad @ w2.T  # (64, 10)*(10, 100) = (64, 100)
    h_grad = h_relu_grad.clone()
    h_grad[h < 0] = 0  # (64, 100)
    # d loss / d h =  2 * (y_pred - y) * w2
    w1_grad = x.T @ h_grad  # (1000, 64) * (64, 100) = (1000, 100)

    # update parameters
    w1 -= learning_rate * w1_grad
    w2 -= learning_rate * w2_grad<jupyter_output>0: loss = 25190348.0
1: loss = 18913532.0
2: loss = 16520530.0
3: loss = 15477824.0
4: loss = 14555248.0
5: loss = 13137495.0
6: loss = 11120706.0
7: loss = 8763305.0
8: loss = 6504405.0
9: loss = 4612281.5
10: loss = 3197467.75
11: loss = 2208325.75
12: loss = 1544545.25
13: loss = 1106569.0
14: loss = 817231.5
15: loss = 623425.9375
16: loss = 490594.25
17: loss = 396976.9375
18: loss = 328826.4375
19: loss = 277561.3125
20: loss = 237781.4375
21: loss = 206080.96875
22: loss = 180245.0
23: loss = 158765.765625
24: loss = 140644.90625
25: loss = 125173.875
26: loss = 111869.390625
27: loss = 100325.265625
28: loss = 90227.8671875
29: loss = 81352.1015625
30: loss = 73515.3828125
31: loss = 66583.5546875
32: loss = 60419.18359375
33: loss = 54922.2890625
34: loss = 50012.57421875
35: loss = 45617.9296875
36: loss = 41671.6953125
37: loss = 38121.78125
38: loss = 34927.96875
39: loss = 32044.8046875
40: loss = 29436.328125
41: loss = 27080.15625
42: loss = 24941.12109375
43: loss = 229[...]<jupyter_text>1.3 Autograd<jupyter_code>x = torch.randn(N, D_IN)
y = torch.randn(N, D_OUT)
w1 = torch.randn(D_IN, H, requires_grad=True)
w2 = torch.randn(H, D_OUT, requires_grad=True)

for epoch in range(NUM_EPOCHS):
    # fowward pass
    y_pred = F.relu(x @ w1) @ w2
    loss = (y_pred - y).pow(2).sum()
    print(f"{epoch}: loss = {loss.item()}")

    # calculate gradients
    loss.backward()

    # update parameters
    with torch.no_grad():
        w1 -= learning_rate * w1.grad
        w2 -= learning_rate * w2.grad
        w1.grad.zero_()
        w2.grad.zero_()<jupyter_output>0: loss = 33568252.0
1: loss = 32938532.0
2: loss = 36839896.0
3: loss = 38404080.0
4: loss = 32796082.0
5: loss = 21467252.0
6: loss = 11210849.0
7: loss = 5359230.0
8: loss = 2764205.5
9: loss = 1680146.875
10: loss = 1189829.0
11: loss = 928319.375
12: loss = 762477.375
13: loss = 642991.6875
14: loss = 550300.6875
15: loss = 475091.375
16: loss = 412737.4375
17: loss = 360368.90625
18: loss = 315947.90625
19: loss = 278006.09375
20: loss = 245425.703125
21: loss = 217308.65625
22: loss = 192968.046875
23: loss = 171802.125
24: loss = 153327.0625
25: loss = 137133.375
26: loss = 122914.1328125
27: loss = 110412.78125
28: loss = 99362.609375
29: loss = 89573.96875
30: loss = 80883.3125
31: loss = 73154.0
32: loss = 66262.671875
33: loss = 60102.44921875
34: loss = 54591.0859375
35: loss = 49650.015625
36: loss = 45209.90625
37: loss = 41220.62890625
38: loss = 37627.390625
39: loss = 34387.2265625
40: loss = 31458.4140625
41: loss = 28811.703125
42: loss = 26413.6875
43: loss = 24239[...]<jupyter_text>1.4 nn<jupyter_code>x = torch.randn(N, D_IN)
y = torch.randn(N, D_OUT)

model = nn.Sequential(nn.Linear(D_IN, H), nn.ReLU(), nn.Linear(H, D_OUT))

loss_fn = nn.MSELoss(reduction="sum")

for epoch in range(NUM_EPOCHS):
    # fowward pass
    y_pred = model(x)
    loss = loss_fn(y_pred, y)
    print(f"{epoch}: loss = {loss.item()}")

    # calculate gradients
    model.zero_grad()
    loss.backward()

    # update parameters
    with torch.no_grad():
        for param in model.parameters():
            param -= learning_rate * param.grad<jupyter_output>0: loss = 711.8518676757812
1: loss = 711.3204956054688
2: loss = 710.790283203125
3: loss = 710.261474609375
4: loss = 709.733642578125
5: loss = 709.206787109375
6: loss = 708.6809692382812
7: loss = 708.1553955078125
8: loss = 707.6304321289062
9: loss = 707.1058959960938
10: loss = 706.5819702148438
11: loss = 706.05859375
12: loss = 705.5360717773438
13: loss = 705.0145874023438
14: loss = 704.49365234375
15: loss = 703.973388671875
16: loss = 703.45361328125
17: loss = 702.9344482421875
18: loss = 702.4159545898438
19: loss = 701.898193359375
20: loss = 701.3818359375
21: loss = 700.8661499023438
22: loss = 700.3511352539062
23: loss = 699.8365478515625
24: loss = 699.3223266601562
25: loss = 698.8087158203125
26: loss = 698.2951049804688
27: loss = 697.7821044921875
28: loss = 697.2693481445312
29: loss = 696.7571411132812
30: loss = 696.2454223632812
31: loss = 695.7342529296875
32: loss = 695.2235717773438
33: loss = 694.71337890625
34: loss = 694.2037963867188
35: loss = 693.[...]<jupyter_text>1.5: optim<jupyter_code>x = torch.randn(N, D_IN)
y = torch.randn(N, D_OUT)

model = nn.Sequential(nn.Linear(D_IN, H), nn.ReLU(), nn.Linear(H, D_OUT))

loss_fn = nn.MSELoss(reduction="sum")
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

for epoch in range(NUM_EPOCHS):
    # fowward pass
    y_pred = model(x)
    loss = loss_fn(y_pred, y)
    print(f"{epoch}: loss = {loss.item()}")

    # calculate gradients
    optimizer.zero_grad()
    loss.backward()

    # update parameters
    optimizer.step()<jupyter_output>0: loss = 630.9755249023438
1: loss = 630.805908203125
2: loss = 630.636474609375
3: loss = 630.4671630859375
4: loss = 630.2977905273438
5: loss = 630.1287231445312
6: loss = 629.959716796875
7: loss = 629.790771484375
8: loss = 629.6217041015625
9: loss = 629.4527587890625
10: loss = 629.2838134765625
11: loss = 629.1148681640625
12: loss = 628.9459838867188
13: loss = 628.777099609375
14: loss = 628.6083374023438
15: loss = 628.4395751953125
16: loss = 628.2708740234375
17: loss = 628.1021728515625
18: loss = 627.9335327148438
19: loss = 627.7650756835938
20: loss = 627.5967407226562
21: loss = 627.4283447265625
22: loss = 627.2601318359375
23: loss = 627.0918579101562
24: loss = 626.9237670898438
25: loss = 626.7559204101562
26: loss = 626.5880126953125
27: loss = 626.4201049804688
28: loss = 626.2523193359375
29: loss = 626.084716796875
30: loss = 625.9171142578125
31: loss = 625.7496337890625
32: loss = 625.58203125
33: loss = 625.41455078125
34: loss = 625.2470703125
35: loss = [...]<jupyter_text>1.6 class<jupyter_code>x = torch.randn(N, D_IN)
y = torch.randn(N, D_OUT)


class Model(torch.nn.Module):
    def __init__(self, D_IN, H, D_OUT):
        super().__init__()
        self.linear1 = torch.nn.Linear(D_IN, H)
        self.linear2 = torch.nn.Linear(H, D_OUT)

    def forward(self, x):
        h_relu = self.linear1(x).clamp(min=0)
        y_pred = self.linear2(h_relu)
        return y_pred


model = Model(D_IN, H, D_OUT)
loss_fn = nn.MSELoss(reduction="sum")
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

for epoch in range(NUM_EPOCHS):
    # fowward pass
    y_pred = model(x)
    loss = loss_fn(y_pred, y)
    print(f"{epoch}: loss = {loss.item()}")

    # calculate gradients
    optimizer.zero_grad()
    loss.backward()

    # update parameters
    optimizer.step()<jupyter_output>0: loss = 679.6824951171875
1: loss = 679.499755859375
2: loss = 679.317138671875
3: loss = 679.134521484375
4: loss = 678.9519653320312
5: loss = 678.7694702148438
6: loss = 678.5872192382812
7: loss = 678.405029296875
8: loss = 678.2227783203125
9: loss = 678.0405883789062
10: loss = 677.8583374023438
11: loss = 677.67626953125
12: loss = 677.494140625
13: loss = 677.3120727539062
14: loss = 677.1300048828125
15: loss = 676.9480590820312
16: loss = 676.76611328125
17: loss = 676.5841674804688
18: loss = 676.4024658203125
19: loss = 676.2205810546875
20: loss = 676.038818359375
21: loss = 675.8571166992188
22: loss = 675.6754760742188
23: loss = 675.4939575195312
24: loss = 675.3124389648438
25: loss = 675.1309204101562
26: loss = 674.949462890625
27: loss = 674.76806640625
28: loss = 674.5867919921875
29: loss = 674.4054565429688
30: loss = 674.2242431640625
31: loss = 674.0431518554688
32: loss = 673.8621826171875
33: loss = 673.6812133789062
34: loss = 673.5003662109375
35: loss = [...]<jupyter_text>2. fizzbuzz game<jupyter_code>torch.cuda.empty_cache()
def fizzbuzz_number(i):
    if i % 15 == 0:
        return 3
    elif i % 5 == 0:
        return 2
    elif i % 3 == 0:
        return 1
    else:
        return 0


def fizzbuzz_nts(i, p):
    return [str(i), "fizz", "buzz", "fizzbuzz"][p]


NUM_DIGIT = 14


def binary(i, digit=NUM_DIGIT):
    return np.array([i >> d & 1 for d in range(digit)])


trX = torch.Tensor([binary(i) for i in range(101, 2 ** NUM_DIGIT)]).cuda()
trY = torch.LongTensor([fizzbuzz_number(i) for i in range(101, 2 ** NUM_DIGIT)]).cuda()
NUM_HIDDEN = 100
model = torch.nn.Sequential(
    torch.nn.Linear(NUM_DIGIT, NUM_HIDDEN),
    torch.nn.ReLU(),
    torch.nn.Linear(NUM_HIDDEN, 4),
    torch.nn.Sigmoid(),
)

if USE_CUDA:
    model = model.cuda()

loss_fn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.05)
BATCH_SIZE = 128
for epoch in range(10000):
    for start in range(0, len(trX), BATCH_SIZE):
        end = start + BATCH_SIZE
        batchX = trX[start:end]
        batchY = trY[start:end]
        y_pred = model(batchX)
        loss = loss_fn(y_pred, batchY)

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    # Find loss on training data
    with torch.no_grad():
        loss = loss_fn(model(trX), trY)
        print("Epoch:", epoch, "Loss:", loss.item())

#     optimizer.zero_grad()
#     loss.backward()
#     optimizer.step()
testX = torch.Tensor([binary(i, NUM_DIGIT) for i in range(1, 101)]).cuda()
with torch.no_grad():
    testY = model(testX)
predictions = zip(range(1, 101), list(testY.max(1)[1].data.tolist()))

print([fizzbuzz_nts(i, x) for (i, x) in predictions])
x = torch.randn(100, 10)
print(
    np.sum(
        testY.cpu().max(1)[1].numpy()
        == np.array([fizzbuzz_number(i) for i in range(1, 101)])
    )
)
testY.cpu().max(1)[1].numpy() == np.array([fizzbuzz_number(i) for i in range(1, 101)])<jupyter_output>93
{'fraction_non_alphanumeric': 0.14230942715381145, 'fraction_numerical': 0.28755074424898514, 'mean_word_length': 3.5325383304940376, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '447824', 'license': 'MIT', 'n_tokens_mistral': 8218, 'n_tokens_neox': 5840, 'n_words': 1350, 'path': 'numpy_to_torch.ipynb', 'repo_name': 'neal2018/torch_learn', 'size': '13302'}