<jupyter_start><jupyter_text>Setup<jupyter_code>x_train, y_train, x_valid, y_valid = get_data()
x_train, y_train, x_valid, y_valid = map(tensor, (x_train, y_train, x_valid, y_valid))
x_train.shape, y_train.shape, x_valid.shape, y_valid.shape
n,m = x_train.shape
c = y_train.max()+1
nh = 50
class Model(nn.Module):
    def __init__(self, n_in, nh, n_out):
        super().__init__()
        self.layers = [nn.Linear(n_in,nh), nn.ReLU(), nn.Linear(nh,n_out)]
        
    def __call__(self, x):
        for l in self.layers: x = l(x)
        return x
model = Model(m, nh, 10)
pred = model(x_train)
pred.shape<jupyter_output><empty_output><jupyter_text>Cross Entropy Loss<jupyter_code>def log_softmax(x): return (x.exp() / x.exp().sum(-1, keepdim = True)).log()
sm_pred = log_softmax(pred)
# making mini test dataset
sm_mini = sm_pred[:3]
y_mini = y_train[:3]
sm_mini, y_mini
sm_mini[[0,1,2], [5,0,4]]
def nll(input, target): return -input[range(input.shape[0]),target].mean()
nll(sm_mini, y_mini)
def log_softmax(x): return x - x.exp().sum(-1, keepdim = True).log()
sm_pred_2 = log_softmax(pred)
test_near(sm_pred, sm_pred_2)
def logsumexp(x):
    a = x.max(-1)[0].unsqueeze(-1)
    return a + (x-a).exp().sum(-1,keepdim=True).log()
def log_softmax(x):
    return x - logsumexp(x)
test_near(sm_pred, log_softmax(pred))
loss = nll(sm_pred, y_train)
test_near(nll(log_softmax(pred), y_train), loss)
test_near(F.cross_entropy(pred, y_train), loss)<jupyter_output><empty_output><jupyter_text>Now we have implemented a pytorch's cross entropy function!!  Basic Training Loop<jupyter_code>loss_func = F.cross_entropy
epochs = 1
lr = 0.5
bs = 64
model = Model(m, nh, 10)
#export
def accuracy(out, yb): return (out.argmax(-1) == yb).float().mean()
xb = x_train[0:bs]
yb = y_train[0:bs]
preds = model(xb)
accuracy(preds, yb)
for epoch in range(epochs):
        for i in range((n-1)//bs + 1):
            start_i = i * bs
            end_i = start_i + bs
            xb = x_train[start_i:end_i]
            yb = y_train[start_i:end_i]
            preds = model(xb)
            loss = loss_func(preds, yb)
            loss.backward()
            with torch.no_grad():
                for l in model.layers:
                    if hasattr(l, 'weight'):
                        l.weight -= lr * l.weight.grad
                        l.bias -= lr * l.bias.grad
                        l.weight.grad.zero_()
                        l.bias  .grad.zero_()
loss_func(model(xb), yb), accuracy(model(xb), yb)<jupyter_output><empty_output><jupyter_text>Using Parameters and Optim<jupyter_code>class Model(nn.Module):
    def __init__(self, n_in, nh, n_out):
        super().__init__()
        self.l1 = nn.Linear(n_in, nh)
        self.l2 = nn.Linear(nh, n_out)
    def __call__(self, x):
        return self.l2(F.relu(self.l1(x)))
        
model = Model(m, nh, 10)
for p in model.parameters(): print(p.shape)
for name,l in model.named_children(): print(f"{name}", l)
model.l1
def fit():
    for epoch in range(epochs):
        for i in range((n-1)//bs + 1):
            start_i = i * bs
            end_i = start_i + bs
            xb = x_train[start_i:end_i]
            yb = y_train[start_i:end_i]
            preds = model(xb)
            loss = loss_func(preds, yb)
            loss.backward()
            with torch.no_grad():
                for p in model.parameters(): p -= lr * p.grad
                model.zero_grad()
                    
fit()
loss_func(model(xb), yb), accuracy(model(xb), yb)<jupyter_output><empty_output><jupyter_text>Registering Modules<jupyter_code>layers = [nn.Linear(m, nh), nn.ReLU(), nn.Linear(nh, 10)]
class Model(nn.Module):
    def __init__(self, layers):
        super().__init__()
        self.layers = layers
        for i,l in enumerate(layers): self.add_module(f'layer{i}', l)
    def __call__(self, x):
        for l in layers: x = l(x)
        return x
model = Model(layers)
model<jupyter_output><empty_output><jupyter_text>nn.ModuleList<jupyter_code>class Model(nn.Module):
    def __init__(self, layers):
        super().__init__()
        self.layers = nn.ModuleList(layers)
    def __call__(self, x):
        for l in layers: x = l(x)
        return x
model = Model(layers)
model
fit()
loss_func(model(xb), yb), accuracy(model(xb), yb)<jupyter_output><empty_output><jupyter_text>nn.Sequential<jupyter_code>model = nn.Sequential(nn.Linear(m,nh), nn.ReLU(), nn.Linear(nh,10))
fit()
loss_func(model(xb), yb), accuracy(model(xb), yb)
model<jupyter_output><empty_output><jupyter_text>Optim<jupyter_code>class Optimizer():
    def __init__(self, params, lr=0.5):
        self.params = list(params)
        self.lr = lr
    def step(self):
        with torch.no_grad():
            for p in self.params: p -= p.grad * self.lr
    def zero_grad(self):
        for p in self.params: p.grad.data.zero_()
opt = Optimizer(model.parameters())
def fit():
    for epoch in range(epochs):
        for i in range((n-1)//bs + 1):
            start_i = i * bs
            end_i = start_i + bs
            xb = x_train[start_i:end_i]
            yb = y_train[start_i:end_i]
            preds = model(xb)
            loss = loss_func(preds, yb)
            
            loss.backward()
            opt.step()
            opt.zero_grad()
                    
fit()
loss_func(model(xb), yb), accuracy(model(xb), yb)
from torch import optim
optim.SGD.step??
def get_model():
    model = nn.Sequential(nn.Linear(m,nh), nn.ReLU(), nn.Linear(nh,10))
    return model, optim.SGD(model.parameters(), lr=lr)
model, opt = get_model()
model, opt
def fit():
    for epoch in range(epochs):
        for i in range((n-1)//bs + 1):
            start_i = i * bs
            end_i = start_i + bs
            xb = x_train[start_i:end_i]
            yb = y_train[start_i:end_i]
            preds = model(xb)
            loss = loss_func(preds, yb)
            
            loss.backward()
            opt.step()
            opt.zero_grad()
                    
fit()
loss_func(model(xb), yb), accuracy(model(xb), yb)<jupyter_output><empty_output><jupyter_text>Dataset Class<jupyter_code>#export
class Dataset():
    def __init__(self, x, y): self.x, self.y= x,y
    def __len__(self): return len(self.x)
    def __getitem__(self, i): return self.x[i], self.y[i]
train_ds = Dataset(x_train, y_train)
valid_ds = Dataset(x_valid, y_valid)
def fit():
    for epoch in range(epochs):
        for i in range((n-1)//bs + 1):
            start_i = i * bs
            end_i = start_i + bs
            xb,yb = train_ds[i*bs : i*bs+bs]
            preds = model(xb)
            loss = loss_func(preds, yb)
            
            loss.backward()
            opt.step()
            opt.zero_grad()
                    
model, opt = get_model()
fit()
loss_func(model(xb), yb), accuracy(model(xb), yb)<jupyter_output><empty_output><jupyter_text>Dataloader Class<jupyter_code>class DataLoader():
    def __init__(self, ds, bs): self.ds, self.bs = ds,bs
    def __iter__(self):
        for i in range(0, len(self.ds),self.bs): yield self.ds[i:i+self.bs]
train_dl = DataLoader(train_ds, bs)
def fit():
    for epoch in range(epochs):
        for xb,yb in train_dl:
            preds = model(xb)
            loss = loss_func(preds, yb)
            
            loss.backward()
            opt.step()
            opt.zero_grad()
                    
model, opt = get_model()
fit()
loss_func(model(xb), yb), accuracy(model(xb), yb)<jupyter_output><empty_output><jupyter_text>Random Sampling<jupyter_code>#export
class Sampler():
    def __init__(self, ds, bs, shuffle=False):
        self.n,self.bs,self.shuffle = len(ds), bs, shuffle
    def __iter__(self):
        self.idxs = torch.randperm(self.n) if self.shuffle else torch.arange(self.n)
        for i in range(0, self.n, self.bs): yield self.idxs[i:i+self.bs]
small_ds = Dataset(*train_ds[:10])
s = Sampler(small_ds, 3, False)
[o for o in s]
s = Sampler(small_ds, 3, True)
[o for o in s]
#export
class DataLoader():
    def __init__(self, ds, s): self.ds, self.s = ds,s
    def __iter__(self):
        for idxs in self.s:
            yield self.ds[idxs]
s = Sampler(train_ds, 64, True)
train_dl = DataLoader(train_ds, s)
def fit():
    for epoch in range(epochs):
        for xb,yb in train_dl:
            preds = model(xb)
            loss = loss_func(preds, yb)
            
            loss.backward()
            opt.step()
            opt.zero_grad()
                    
model, opt = get_model()
fit()
loss_func(model(xb), yb), accuracy(model(xb), yb)
train_samp = Sampler(train_ds, 64, True)
valid_samp = Sampler(valid_ds, 64, False)
train_dl = DataLoader(train_ds, train_samp)
valid_dl = DataLoader(valid_ds, valid_samp)
xb,yb = next(iter(valid_dl))
plt.imshow(xb[0].view(28,28), cmap='gray')
yb[0]
xb,yb = next(iter(train_dl))
plt.imshow(xb[0].view(28,28), cmap='gray')
yb[0]<jupyter_output><empty_output><jupyter_text>PyTorch Dataloader<jupyter_code>#export
from torch.utils.data import DataLoader, SequentialSampler, RandomSampler
def collate(b):
    xs,ys = zip(*b)
    return torch.stack(xs),torch.stack(ys)
train_dl = DataLoader(train_ds, bs, sampler=RandomSampler(train_ds), collate_fn=collate)
valid_dl = DataLoader(valid_ds, bs, sampler=SequentialSampler(valid_ds), collate_fn=collate)
model,opt = get_model()
fit()
loss_func(model(xb), yb), accuracy(model(xb), yb)
train_dl = DataLoader(train_ds, bs, shuffle=True, drop_last=True)
valid_dl = DataLoader(valid_ds, bs, shuffle=False)
model,opt = get_model()
fit()

loss,acc = loss_func(model(xb), yb), accuracy(model(xb), yb)
assert acc>0.7
loss,acc
def fit(epochs, model, loss_func, opt, train_dl, valid_dl):
    for epoch in range(epochs):
        model.train()
        for xb,yb in train_dl:
            loss = loss_func(model(xb), yb)
            loss.backward()
            opt.step()
            opt.zero_grad()
            
        model.eval()
        with torch.no_grad():
            tot_loss, tot_acc = 0.,0.
            for xb,yb in valid_dl:
                pred = model(xb)
                tot_loss += loss_func(pred, yb)
                tot_acc += accuracy(pred, yb)
            nv = len(valid_dl)
            print(epoch, tot_loss/nv, tot_acc/nv)
    return tot_loss/nv, tot_acc/nv
#export
def get_dls(train_ds, valid_ds, bs, **kwargs):
    return (DataLoader(train_ds, batch_size=bs, shuffle=True, **kwargs),
            DataLoader(valid_ds, batch_size=bs*2, **kwargs))
train_dl,valid_dl = get_dls(train_ds, valid_ds, bs)
model,opt = get_model()
loss,acc = fit(5, model, loss_func, opt, train_dl, valid_dl)
!python nb2script.py minibatch_training.ipynb
!cat exp/nb_minibatch.py<jupyter_output>#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# file to edit: dev_nb/minibatch_training.ipynb

def accuracy(out, yb): return (out.argmax(-1) == yb).float().mean()

class Dataset():
    def __init__(self, x, y): self.x, self.y= x,y
    def __len__(self): return len(self.x)
    def __getitem__(self, i): return self.x[i], self.y[i]

class Sampler():
    def __init__(self, ds, bs, shuffle=False):
        self.n,self.bs,self.shuffle = len(ds), bs, shuffle
    def __iter__(self):
        self.idxs = torch.randperm(self.n) if self.shuffle else torch.arange(self.n)
        for i in range(0, self.n, self.bs): yield self.idxs[i:i+self.bs]

class DataLoader():
    def __init__(self, ds, s): self.ds, self.s = ds,s
    def __iter__(self):
        for idxs in self.s:
            yield self.ds[idxs]

from torch.utils.data import DataLoader, SequentialSampler, RandomSample[...]
{'fraction_non_alphanumeric': 0.14940547992417716, 'fraction_numerical': 0.007754609684645873, 'mean_word_length': 2.825642715886618, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '543333', 'license': 'MIT', 'n_tokens_mistral': 4382, 'n_tokens_neox': 4259, 'n_words': 914, 'path': 'minibatch_training.ipynb', 'repo_name': 'KyleMoore1/fastai', 'size': '11606'}