<jupyter_start><jupyter_text>A basic training loop  MNIST data setup<jupyter_code>from pathlib import Path

DATA_PATH = Path('data')
PATH = DATA_PATH/'mnist'

PATH.mkdir(parents=True, exist_ok=True)
import requests

URL='http://deeplearning.net/data/mnist/'
FILENAME='mnist.pkl.gz'

if not (PATH/FILENAME).exists():
    content = requests.get(URL+FILENAME).content
    (PATH/FILENAME).open('wb').write(content)
import pickle, gzip

((x, y), (x_valid, y_valid), _) = pickle.load(gzip.open(PATH/FILENAME, 'rb'), encoding='latin-1')
import torch 

x,y,x_valid,y_valid = map(torch.tensor, (x,y,x_valid,y_valid))
n,c = x.shape
x, x.shape, y.min(), y.max()<jupyter_output><empty_output><jupyter_text>Basic model and training loop<jupyter_code>import math

weights = torch.randn(784,10)/math.sqrt(784)
weights.requires_grad_()
bias = torch.zeros(10, requires_grad=True)
import torch.nn.functional as F

def model(xb):
    xb = (xb @ weights) + bias
    return F.log_softmax(xb, dim=-1)
bs=64
preds = model(x[0:bs])
preds[0], preds.shape
loss_fn = F.nll_loss
loss_fn(preds, y[0:bs])
lr = 0.5
epochs = 2
from IPython.core.debugger import set_trace
for epoch in range(epochs):
    for i in range((n-1)//bs + 1):
#         set_trace()
        start_i = i*bs
        end_i = min(start_i+bs, n)
        xb = x[start_i:end_i]
        yb = y[start_i:end_i]
        pred = model(xb)
        loss = loss_fn(pred, yb)

        loss.backward()
        with torch.no_grad():
            weights -= weights.grad * lr
            bias -= bias.grad * lr
            weights.grad.zero_()
            bias.grad.zero_()
loss_fn(model(x[0:bs]), y[0:bs])<jupyter_output><empty_output><jupyter_text>Refactor using nn.Module<jupyter_code>from torch import nn

class Mnist_Logistic(nn.Module):
    def __init__(self):
        super().__init__()
        self.weights = nn.Parameter(torch.randn(784,10)/math.sqrt(784))
        self.bias = nn.Parameter(torch.zeros(10))

    def forward(self, xb):
        xb = (xb @ self.weights) + self.bias
        return F.log_softmax(xb, dim=-1)
model = Mnist_Logistic()
loss_fn(model(x[0:bs]), y[0:bs])
for epoch in range(epochs):
    for i in range((n-1)//bs + 1):
        start_i = i*bs
        end_i = min(start_i+bs, n)
        xb = x[start_i:end_i]
        yb = y[start_i:end_i]
        pred = model(xb)
        loss = loss_fn(pred, yb)

        loss.backward()
        with torch.no_grad():
            for p in model.parameters(): p -= p.grad * lr
            model.zero_grad()
loss_fn(model(x[0:bs]), y[0:bs])<jupyter_output><empty_output><jupyter_text>Refactor using nn.Linear<jupyter_code>class Mnist_Logistic(nn.Module):
    def __init__(self):
        super().__init__()
        self.lin = nn.Linear(784,10)

    def forward(self, xb):
        return F.log_softmax(self.lin(xb), dim=-1)
model = Mnist_Logistic()
loss_fn(model(x[0:bs]), y[0:bs])
for epoch in range(epochs):
    for i in range((n-1)//bs + 1):
        start_i = i*bs
        end_i = min(start_i+bs, n)
        xb = x[start_i:end_i]
        yb = y[start_i:end_i]
        pred = model(xb)
        loss = loss_fn(pred, yb)

        loss.backward()
        with torch.no_grad():
            for p in model.parameters(): p -= p.grad * lr
            model.zero_grad()
loss_fn(model(x[0:bs]), y[0:bs])<jupyter_output><empty_output><jupyter_text>Refactor using optim<jupyter_code>from torch import optim
model = Mnist_Logistic()
opt = optim.SGD(model.parameters(), lr=1.)

loss_fn(model(x[0:bs]), y[0:bs])
for epoch in range(epochs):
    for i in range((n-1)//bs + 1):
        start_i = i*bs
        end_i = min(start_i+bs, n)
        xb = x[start_i:end_i]
        yb = y[start_i:end_i]
        pred = model(xb)
        loss = loss_fn(pred, yb)

        loss.backward()
        opt.step()
        opt.zero_grad()
loss_fn(model(x[0:bs]), y[0:bs])<jupyter_output><empty_output><jupyter_text>Refactor using Dataset<jupyter_code>from torch.utils.data import TensorDataset
model = Mnist_Logistic()
opt = optim.SGD(model.parameters(), lr=1.)
train_ds = TensorDataset(x, y)
loss_fn(model(x[0:bs]), y[0:bs])
for epoch in range(epochs):
    for i in range((n-1)//bs + 1):
        xb,yb = train_ds[i*bs : i*bs+bs]
        pred = model(xb)
        loss = loss_fn(pred, yb)

        loss.backward()
        opt.step()
        opt.zero_grad()
loss_fn(model(x[0:bs]), y[0:bs])<jupyter_output><empty_output><jupyter_text>Refactor using DataLoader<jupyter_code>from torch.utils.data import DataLoader
model = Mnist_Logistic()
opt = optim.SGD(model.parameters(), lr=1.)
train_ds = TensorDataset(x, y)
train_dl = DataLoader(train_ds, batch_size=bs)
loss_fn(model(x[0:bs]), y[0:bs])
for epoch in range(epochs):
    for xb,yb in train_dl:
        pred = model(xb)
        loss = loss_fn(pred, yb)

        loss.backward()
        opt.step()
        opt.zero_grad()
loss_fn(model(x[0:bs]), y[0:bs])<jupyter_output><empty_output><jupyter_text>Add validation  First try<jupyter_code>model = Mnist_Logistic()
opt = optim.SGD(model.parameters(), lr=1.)
train_ds = TensorDataset(x, y)
train_dl = DataLoader(train_ds, batch_size=bs, shuffle=True)
valid_ds = TensorDataset(x_valid, y_valid)
valid_dl = DataLoader(valid_ds, batch_size=bs*2)
loss_fn(model(x_valid[0:bs]), y_valid[0:bs])
for epoch in range(epochs):
    model.train()
    for xb,yb in train_dl:
        pred = model(xb)
        loss = loss_fn(pred, yb)

        loss.backward()
        opt.step()
        opt.zero_grad()
        
    model.eval()
    valid_loss = 0.
    with torch.no_grad():
        for xb,yb in valid_dl:
            valid_loss += loss_fn(model(xb), yb)

    print(epoch, valid_loss/len(valid_dl))<jupyter_output>0 tensor(0.3758)
1 tensor(0.3508)<jupyter_text>Create fit()<jupyter_code>def loss_batch(model, xb, yb, opt=None):
    loss = loss_fn(model(xb), yb)

    if opt is not None:
        loss.backward()
        opt.step()
        opt.zero_grad()
        
    return loss.item(), len(xb)
import numpy as np

def fit(epochs, model, loss_fn, opt, train_dl, valid_dl):
    for epoch in range(epochs):
        model.train()
        for xb,yb in train_dl: loss_batch(model, xb, yb, opt)

        model.eval()
        with torch.no_grad():
            losses,nums = zip(*[loss_batch(model, xb, yb)
                                for xb,yb in valid_dl])
        val_loss = np.sum(np.multiply(losses,nums)) / np.sum(nums)

        print(epoch, val_loss)
model = Mnist_Logistic()
opt = optim.SGD(model.parameters(), lr=0.5)
train_ds = TensorDataset(x, y)
train_dl = DataLoader(train_ds, batch_size=bs, shuffle=True)

valid_ds = TensorDataset(x_valid, y_valid)
valid_dl = DataLoader(valid_ds, batch_size=bs*2)
fit(epochs, model, loss_fn, opt, train_dl, valid_dl)<jupyter_output>0 0.26180660036802295
1 0.26372072726488116<jupyter_text>Move opt creation into fit()<jupyter_code>def fit(epochs, model, loss_fn, opt_fn, lr, train_dl, valid_dl):
    opt = opt_fn(model.parameters(), lr=lr)

    for epoch in range(epochs):
        model.train()
        for xb,yb in train_dl: loss_batch(model, xb, yb, opt)

        model.eval()
        with torch.no_grad():
            losses,nums = zip(*[loss_batch(model, xb, yb)
                                for xb,yb in valid_dl])
        val_loss = np.sum(np.multiply(losses,nums)) / np.sum(nums)

        print(epoch, val_loss)
model = Mnist_Logistic()
train_ds = TensorDataset(x, y)
train_dl = DataLoader(train_ds, batch_size=bs, shuffle=True)

valid_ds = TensorDataset(x_valid, y_valid)
valid_dl = DataLoader(valid_ds, batch_size=bs*2)
fit(epochs, model, loss_fn, optim.SGD, lr, train_dl, valid_dl)<jupyter_output>0 0.29762732087373733
1 0.2890522579073906<jupyter_text>Refactor to ModelData<jupyter_code>class ModelData():
    def __init__(self, train_ds, valid_ds, bs):
        self.train_dl = DataLoader(train_ds, batch_size=bs, shuffle=True)
        self.valid_dl = DataLoader(valid_ds, batch_size=bs*2)
def fit(epochs, model, loss_fn, opt_fn, lr, data):
    opt = opt_fn(model.parameters(), lr=lr)

    for epoch in range(epochs):
        model.train()
        for xb,yb in data.train_dl:
            loss_batch(model, xb, yb, opt)

        model.eval()
        with torch.no_grad():
            losses,nums = zip(*[loss_batch(model, xb, yb)
                                for xb,yb in data.valid_dl])
        val_loss = np.sum(np.multiply(losses,nums)) / np.sum(nums)

        print(epoch, val_loss)
model = Mnist_Logistic()
data = ModelData(TensorDataset(x, y), TensorDataset(x_valid, y_valid), bs)
fit(epochs, model, loss_fn, optim.SGD, lr, data)<jupyter_output>0 0.4939220434188843
1 0.302523850440979<jupyter_text>Switch to CNN  First try<jupyter_code>class Mnist_CNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1,  16, kernel_size=3, stride=2, padding=1)
        self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1)
        self.conv3 = nn.Conv2d(16, 10,  kernel_size=3, stride=2, padding=1)

    def forward(self, xb):
        xb = xb.view(-1,1,28,28)
        xb = F.relu(self.conv1(xb))
        xb = F.relu(self.conv2(xb))
        xb = F.relu(self.conv3(xb))
        xb = F.avg_pool2d(xb, 4)
        xb = xb.view(-1,xb.size(1))
        return F.log_softmax(xb, dim=-1)
model = Mnist_CNN()
fit(epochs, model, loss_fn, optim.SGD, lr*2, data)
fit(epochs, model, loss_fn, optim.SGD, lr/2, data)<jupyter_output>0 0.41251860780715943
1 0.4114646418571472<jupyter_text>Transformation<jupyter_code>def mnist2image(v): return v.view(1,28,28)
from torch.utils.data import Dataset

class TransformedDataset(Dataset):
    def __init__(self, ds, x_tfms=None, y_tfms=None):
        self.ds,self.x_tfms,self.y_tfms = ds,x_tfms,y_tfms
        
    def __len__(self): return len(self.ds)

    def __getitem__(self,i):
        x,y = self.ds[i]
        if self.x_tfms is not None: x = self.x_tfms(x)
        if self.y_tfms is not None: y = self.y_tfms(y)
        return x,y
train_tfm_ds = TransformedDataset(train_ds, mnist2image)
valid_tfm_ds = TransformedDataset(valid_ds, mnist2image)
train_dl = DataLoader(train_tfm_ds, batch_size=bs, shuffle=True)
valid_dl = DataLoader(valid_tfm_ds, batch_size=bs*2)
data = ModelData(train_tfm_ds, valid_tfm_ds, bs)
class Mnist_CNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1,  16, kernel_size=3, stride=2, padding=1)
        self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=1)
        self.conv3 = nn.Conv2d(16, 10,  kernel_size=3, stride=2, padding=1)

    def forward(self, xb):
        xb = F.relu(self.conv1(xb))
        xb = F.relu(self.conv2(xb))
        xb = F.relu(self.conv3(xb))
        xb = F.avg_pool2d(xb, 4)
        xb = xb.view(-1,xb.size(1))
        return F.log_softmax(xb, dim=-1)
model = Mnist_CNN()
fit(epochs, model, loss_fn, optim.SGD, lr*2, data)
fit(epochs, model, loss_fn, optim.SGD, lr/2, data)<jupyter_output>0 0.4676215552330017
1 0.46685396366119386<jupyter_text>Refactor network<jupyter_code>class Simple_CNN(nn.Module):
    def __init__(self, actns, kernel_szs, strides):
        super().__init__()
        self.convs = nn.ModuleList([
            nn.Conv2d(actns[i], actns[i+1], kernel_szs[i], stride=strides[i], padding=kernel_szs[i]//2)
            for i in range(len(strides))])

    def forward(self, xb):
        for conv in self.convs: xb = F.relu(conv(xb))
        xb = F.adaptive_avg_pool2d(xb, 1)
        xb = xb.view(-1,xb.size(1))
        return F.log_softmax(xb, dim=-1)
def get_model(): return Simple_CNN([1,16,16,10], [3,3,3], [2,2,2])
model = get_model()
fit(epochs, model, loss_fn, optim.SGD, lr*2, data)
fit(epochs, model, loss_fn, optim.SGD, lr/2, data)<jupyter_output>0 0.18453329709768296
1 0.1730178252518177<jupyter_text>CUDA<jupyter_code>default_device = torch.device('cuda')
class DeviceDataLoader():
    def __init__(self, dl, device):
        self.dl,self.device = dl,device
        
    def __iter__(self):
        for b in self.dl: yield [o.to(device) for o in b]

class ModelData():
    def __init__(self, train_ds, valid_ds, bs, device):
        train_dl = DataLoader(train_tfm_ds, batch_size=bs, shuffle=True)
        self.train_dl = DeviceDataLoader(train_dl, device)
        valid_dl = DataLoader(valid_tfm_ds, batch_size=bs*2)
        self.valid_dl = DeviceDataLoader(valid_dl, device)
data = ModelData(TensorDataset(x, y), TensorDataset(x_valid, y_valid),
                 bs, default_device)
model = Mnist_CNN().to(default_device)
fit(epochs, model, loss_fn, optim.SGD, lr*2, data)
fit(epochs, model, loss_fn, optim.SGD, lr/2, data)<jupyter_output>0 0.22181436491012574
1 0.21518816499710083
{'fraction_non_alphanumeric': 0.142074332909784, 'fraction_numerical': 0.037007623888182975, 'mean_word_length': 3.3939288206559666, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '181625', 'license': 'Apache-2.0', 'n_tokens_mistral': 5229, 'n_tokens_neox': 4986, 'n_words': 938, 'path': 'dev_nb/001_training.ipynb', 'repo_name': 'sgugger/fastai_v1', 'size': '12592'}