<jupyter_start><jupyter_text>Deep Learning with PyTorch: A 60 Minute Blitz---<jupyter_code>import torch

torch.__version__<jupyter_output><empty_output><jupyter_text>Getting Started  Tensors<jupyter_code>x = torch.empty(5,4)
print(x)
x = torch.rand(5,4)
x
x = torch.zeros(5, 7, dtype=torch.long)
print(x)
x = torch.tensor([5, 4], dtype=torch.int)
x
x = x.new_ones(5,1)
x
x = torch.randn_like(x, dtype=torch.float)
x
x.size()<jupyter_output><empty_output><jupyter_text>Operations<jupyter_code>x = torch.zeros(3,2)
y = torch.ones(3,2)
x+y
torch.add(x,y)
z = torch.empty(3,2)
torch.add(x,y, out=z)
z
y.add_(x)
y
y.t_()
y = torch.randn(3,2)
y
y[:,1], y[1,:]
x = torch.randn(5,6)
y = x.view(30)
z = x.view(-1, 2)
x.shape, y.shape, z.shape
x = torch.randn(1)
x.item()<jupyter_output><empty_output><jupyter_text>More operations at: [http://pytorch.org/docs/torch]  NumPy Bridge<jupyter_code>a = torch.ones(5)
a
b = a.numpy()
b
a.add_(1)
a, b
import numpy as np
a = np.ones(5)
b = torch.from_numpy(a)
a,b
np.add(a, 1, out=a)
a, b
a = a + 1
a, b<jupyter_output><empty_output><jupyter_text>CUDA Tensors<jupyter_code>torch.cuda.is_available()
dev = torch.device("cuda")
x = torch.ones(2,3, device=dev)
x
y = torch.rand(2,3)
y = y.to(dev)
y
z = x+y
z
z = z.to("cpu", dtype=torch.int)
z<jupyter_output><empty_output><jupyter_text>Autograd: Automatic Differentiation  Tensor<jupyter_code>a = torch.tensor([1,2,3], requires_grad=True, dtype=torch.float)
a
b = a * 3
b
c = b + 10
c
b.grad_fn, c.grad_fn
out = c.sum()
out
out.grad_fn, out.requires_grad<jupyter_output><empty_output><jupyter_text>Gradient ```d(out)/dx = (d(out)/dz) * (dz/dy) * (dy/dx)```<jupyter_code>a = torch.tensor([1,1,1], dtype=torch.float, requires_grad=True)
b = torch.tensor([2,2,2], dtype=torch.float, requires_grad=True)
a, b
c = a*a + b
d = c.mean()
d
d.backward(torch.tensor([1.]))
a.grad, b.grad, c.grad
x = torch.ones([2,2], requires_grad=True)
x
y = x + 2
y
z = y*y * 3
z
out = z.sum()
out
out.backward()
out.grad, z.grad, y.grad, x.grad<jupyter_output><empty_output><jupyter_text>Dynamic gradient<jupyter_code>x = torch.ones(3, requires_grad=True)
x
y = x*2
while y.sum() < 20:
    print(y)
    y = y*2
y
y.backward(torch.tensor([1,1,1]))
y.grad, x.grad<jupyter_output><empty_output><jupyter_text>More at: [https://pytorch.org/docs/stable/autograd.html]  Neural Networks  Build Model<jupyter_code>import torch.nn as nn
import torch.nn.functional as F
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # 1 input channel, 6 output channel
        # --- Kernel ---
        # 5x5 square convolutions
        self.conv1 = nn.Conv2d(in_channels=1, out_channels=6, kernel_size=(5,5))
        self.conv2 = nn.Conv2d(in_channels=6, out_channels=16, kernel_size=(5,5))
        # --- Affine Transformation: y = xW + b ---
        self.fc1 = nn.Linear(in_features=16*5*5, out_features=120)
        self.fc2 = nn.Linear(in_features=120, out_features=84)
        self.fc3 = nn.Linear(in_features=84, out_features=10)

    def forward(self, x):
        # --- CNN ---
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, kernel_size=(2,2))
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, kernel_size=(2,2))
        # --- FCC ---
        x = x.view(-1, self.num_flat_features(x))
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

    def num_flat_features(self, x):
        # all dimensions except batch dimension
        size = x.size()[1:]
        num_features = 1
        for s in size:
            num_features *= s
        return num_features
    
md = Net()
md
params = list(md.parameters())
len(params), params[0].shape, params[9]<jupyter_output><empty_output><jupyter_text>**nn.Conv2d will take in a 4D Tensor of nSamples x nChannels x Height x Width**<jupyter_code>x1 = torch.rand(1,1,32,32)
x1
y_pred = md(x1)
y_pred<jupyter_output><empty_output><jupyter_text>Loss<jupyter_code>y.shape
y_true = torch.arange(1, 11)
y_true = y_true.view(1, -1)
y_true.shape
criterion = nn.MSELoss()
loss = criterion(y_pred, y_true)
loss
loss.grad_fn
print(loss.grad_fn)
print(loss.grad_fn.next_functions)
print(loss.grad_fn.next_functions[0][0].next_functions)
print(loss.grad_fn.next_functions[0][0].next_functions[1][0].next_functions)<jupyter_output><MseLossBackward object at 0x7fc52d1e1358>
((<AddmmBackward object at 0x7fc52d0b74a8>, 0),)
((<ExpandBackward object at 0x7fc52d0b7550>, 0), (<ReluBackward object at 0x7fc52d0b75c0>, 0), (<TBackward object at 0x7fc52d0b75f8>, 0))
((<AddmmBackward object at 0x7fc52d0b75c0>, 0),)<jupyter_text>Backprop<jupyter_code>md.zero_grad()
md.conv1.weight.grad, md.conv1.bias.grad
loss.backward()
md.conv1.bias.grad
md.conv1.bias.data<jupyter_output><empty_output><jupyter_text>**More Network Modules: [https://pytorch.org/docs/stable/nn.html]**  Update Weights<jupyter_code>lr = 0.01
for f in md.parameters():
    f.data = f.data - lr * f.grad.data
md.conv1.bias.data
-0.1178 + 0.0015*0.01
import torch.optim as optim

optimizer = optim.SGD(md.parameters(), lr=0.01)
criterion = nn.MSELoss()
for i in range(10):
    y_pred = md(x1)
    loss = criterion(y_pred, y_true)
    loss.backward()
    optimizer.step()
    print(md.conv1.bias.data)<jupyter_output>tensor([ 0.1933, -0.1284, -0.1433, -0.1972,  0.0596,  0.0186])
tensor([ 0.1951, -0.1259, -0.1395, -0.1992,  0.0602,  0.0216])
tensor([ 0.2003, -0.1215, -0.1322, -0.2011,  0.0627,  0.0282])
tensor([ 0.2139, -0.1134, -0.1169, -0.2021,  0.0706,  0.0432])
tensor([ 0.2559, -0.0970, -0.0769, -0.2014,  0.0961,  0.0844])
tensor([ 0.3011, -0.0799, -0.0343, -0.2006,  0.1236,  0.1283])
tensor([-2.1458, -0.5695, -2.0483, -0.3465, -1.3616, -1.9554])
tensor([-4.5928, -1.0590, -4.0623, -0.4925, -2.8469, -4.0391])
tensor([-7.0397, -1.5485, -6.0763, -0.6385, -4.3321, -6.1228])
tensor([-9.4866, -2.0380, -8.0903, -0.7845, -5.8174, -8.2065])<jupyter_text>Training a Classifier  Loading & Normalizaing CIFAR10<jupyter_code>import torchvision
import torchvision.transforms as transforms<jupyter_output><empty_output><jupyter_text>--- TODO ---  Normalize with mean 0 and std 1<jupyter_code>trnsfrm = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean=(0.5,0.5,0.5), std=(0.5,0.5,0.5))])

trnset = torchvision.datasets.CIFAR10(root="./data", train=True, transform=trnsfrm, download=True)
trnldr = torch.utils.data.DataLoader(dataset=trnset, batch_size=4, shuffle=True, num_workers=4)
testset = torchvision.datasets.CIFAR10(root="./data", train=False, transform=trnsfrm, download=True)
testldr = torch.utils.data.DataLoader(dataset=testset, batch_size=4, shuffle=True, num_workers=4)
classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')<jupyter_output><empty_output><jupyter_text>Data Exploration<jupyter_code>import numpy as np
import matplotlib.pyplot as plt

% matplotlib inline
smpl = iter(trnldr)
imgs, labels = next(smpl)

imgs.shape, labels.shape
def imshow(img):
    img = img/2 + 0.5
    img = img.numpy()
    plt.imshow(np.transpose(img, (1,2,0)))
imshow(torchvision.utils.make_grid(imgs))<jupyter_output><empty_output><jupyter_text>Build CNN<jupyter_code>class CNN(nn.Module):
    
    def __init__(self):
        super(CNN, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16*5*5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)
    
    def forward(self, x):
        # --- Kernel ---
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2)
        # --- FCC ---
        x = x.view(-1, 16*5*5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x
md = CNN()<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>epochs = 2
lr = 0.001
m = 0.9
intrvl = 2000
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(md.parameters(), lr=lr, momentum=m)
for epoch in range(epochs):
    running_loss = 0.0
    for i, data in enumerate(trnldr, 0):
        # get input
        inputs, labels = data
        
        # zero the gradients
        optimizer.zero_grad()
        
        # training step
        outputs = md(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        
        # print status
        running_loss += loss.item()
        if i % 2000 == 1999:
            print("Epoch:{} Iteration:{} Loss:{}".format(epoch, i, running_loss/intrvl))
            running_loss = 0.0

print("Training Complete!")<jupyter_output>Epoch:0 Iteration:1999 Loss:2.228038570046425
Epoch:0 Iteration:3999 Loss:1.894750807851553
Epoch:0 Iteration:5999 Loss:1.6732989878952502
Epoch:0 Iteration:7999 Loss:1.565858641549945
Epoch:0 Iteration:9999 Loss:1.5250832606628537
Epoch:0 Iteration:11999 Loss:1.4591481544449925
Epoch:1 Iteration:1999 Loss:1.393994118154049
Epoch:1 Iteration:3999 Loss:1.3773583032637835
Epoch:1 Iteration:5999 Loss:1.3524968849048018
Epoch:1 Iteration:7999 Loss:1.3120768364071846
Epoch:1 Iteration:9999 Loss:1.2891160056814552
Epoch:1 Iteration:11999 Loss:1.293164505816996
Training Complete!<jupyter_text>Testing<jupyter_code>smpl = iter(testldr)
imgs, labels = next(smpl)
imshow(torchvision.utils.make_grid(imgs))
[classes[i] for i in labels.numpy()]
outputs = md(imgs)
outputs = outputs.detach().numpy()
[classes[np.argmax(i)] for i in outputs]
with torch.no_grad():
    y_true = np.array([])
    y_pred = np.array([])
    for data in testldr:
        imgs, labels = data
        y_true = np.append(y_true, labels.numpy())
        outputs = md(imgs)
        _, pred = torch.max(outputs, 1)
        y_pred = np.append(y_pred, pred.numpy())
        
y_true.shape, y_pred.shape
y_true[:5], y_pred[:5]
acc = (y_true == y_pred).mean()
acc<jupyter_output><empty_output><jupyter_text>Training on GPU<jupyter_code>device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
md = md.to(device)
epochs = 2
lr = 0.001
m = 0.9
intrvl = 2000
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(md.parameters(), lr=lr, momentum=m)
for epoch in range(epochs):
    running_loss = 0.0
    for i, data in enumerate(trnldr, 0):
        # send input to gpu
        inputs, labels = data
        inputs, labels = inputs.to(device), labels.to(device)
        
        # zero the gradients
        optimizer.zero_grad()
        
        # training step
        outputs = md(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        
        # print status
        running_loss += loss.item()
        if i % 2000 == 1999:
            print("Epoch:{} Iteration:{} Loss:{}".format(epoch, i, running_loss/intrvl))
            running_loss = 0.0

print("Training Complete!")<jupyter_output>Epoch:0 Iteration:1999 Loss:1.2099936809688807
Epoch:0 Iteration:3999 Loss:1.2139885279685259
Epoch:0 Iteration:5999 Loss:1.2062833626568317
Epoch:0 Iteration:7999 Loss:1.2033072304725647
Epoch:0 Iteration:9999 Loss:1.1818215675503017
Epoch:0 Iteration:11999 Loss:1.1807507127821446
Epoch:1 Iteration:1999 Loss:1.1087059390246867
Epoch:1 Iteration:3999 Loss:1.1152383999377489
Epoch:1 Iteration:5999 Loss:1.1117257242947818
Epoch:1 Iteration:7999 Loss:1.1121124037504195
Epoch:1 Iteration:9999 Loss:1.082045434758067
Epoch:1 Iteration:11999 Loss:1.1056893708258868
Training Complete!<jupyter_text>Testing<jupyter_code>with torch.no_grad():
    y_true = np.array([])
    y_pred = np.array([])
    for data in testldr:
        imgs, labels = data
        imgs, labels = imgs.to(device), labels.to(device)
        y_true = np.append(y_true, labels.cpu().numpy())
        outputs = md(imgs)
        _, pred = torch.max(outputs, 1)
        y_pred = np.append(y_pred, pred.cpu().numpy())
        
y_true.shape, y_pred.shape
y_true[:5], y_pred[:5]
acc = (y_true == y_pred).mean()
acc<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15724795184751714, 'fraction_numerical': 0.09822772111687009, 'mean_word_length': 3.781374900079936, 'pattern_counts': {'":': 0, '<': 88, '<?xml version=': 0, '>': 87, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '526486', 'license': 'MIT', 'n_tokens_mistral': 5525, 'n_tokens_neox': 4955, 'n_words': 902, 'path': 'notebooks/PyTorch_60mins.ipynb', 'repo_name': 'AmanDaVinci/DeepLabs', 'size': '11962'}