<jupyter_start><jupyter_text>First training: SGD with 1cycle<jupyter_code>def accuracy(out, yb):
    preds = torch.max(out, dim=1)[1]
    return (preds==yb).float().mean()
from typing import Callable, List
@dataclass
class Learner():
    
    loss_fn: Callable = F.cross_entropy
    opt_fn: Callable = optim.SGD
    metrics: List = None
    
    def __init__(self, data, model):
        self.data,self.model = data,model.to(data.device)

    def fit(self, epochs, lr, wd=0, callbacks=None):
        self.opt = HPOptimizer(self.model.parameters(), self.opt_fn, init_lr=lr)
        self.opt.wd = wd
        self.recorder = Recorder(self.opt, self.data.train_dl)
        callbacks.insert(0, self.recorder)
        fit(epochs, self.model, self.loss_fn, self.opt, self.data, callbacks=callbacks, metrics=self.metrics)
        
    def lr_find(self, start_lr=1e-5, end_lr=10, num_it=200):
        cb = LRFinder(self, start_lr, end_lr, num_it)
        a = int(np.ceil(num_it/len(self.data.train_dl)))
        self.fit(a, start_lr, callbacks=[cb])
#export
def loss_batch(model, xb, yb, loss_fn, opt=None, cb_handler=None, metrics=None):
    out = model(xb)
    loss = loss_fn(out, yb)
    mets = [f(out,yb).item() for f in metrics] if metrics is not None else []
    
    if opt is not None:
        if cb_handler is not None: loss = cb_handler.on_backward_begin(loss, out)
        loss.backward()
        if cb_handler is not None: cb_handler.on_backward_end()
        opt.step()
        if cb_handler is not None: cb_handler.on_step_end()
        opt.zero_grad()
        
    return (loss.item(),) + tuple(mets) + (len(xb),)
#export
def fit(epochs, model, loss_fn, opt, data, callbacks=None, metrics=None):
    
    cb_handler = CallbackHandler(callbacks)
    cb_handler.on_train_begin()
    
    for epoch in tnrange(epochs):
        model.train()
        cb_handler.on_epoch_begin()
        
        for xb,yb in data.train_dl:
            xb, yb = cb_handler.on_batch_begin(xb, yb)
            loss,_ = loss_batch(model, xb, yb, loss_fn, opt, cb_handler)
            if cb_handler.on_batch_end(loss): break
        
        if hasattr(data,'valid_dl') and data.valid_dl is not None:
            model.eval()
            with torch.no_grad():
                *val_metrics,nums = zip(*[loss_batch(model, xb, yb, loss_fn, metrics=metrics)
                                for xb,yb in data.valid_dl])
            val_metrics = [np.sum(np.multiply(val,nums)) / np.sum(nums) for val in val_metrics]
            
        else: val_metrics=None
        if cb_handler.on_epoch_end(val_metrics): break
        
    cb_handler.on_train_end()
class TrueWD(Callback):
    
    def __init__(self, learn, wd):
        self.learn,self.wd = learn,wd
        
    def on_train_begin(self):
        self.opt = self.learn.opt
        self.opt.wd = 0
        
    def on_backward_end(self):
        for pg in self.opt.opt.param_groups:
            for p in pg['params']:
                p.data.mul_(1 - self.wd * pg['lr'])
model = Darknet([1, 2, 4, 6, 3], num_classes=len(classes), nf=16).cuda()
learn = Learner(data, model)
learn.loss_fn = F.cross_entropy
learn.metrics = [accuracy]
learn.opt_fn = partial(optim.Adam, betas=(0.95,0.99))
learn.lr_find()
learn.recorder.plot()
scheds = [OneCycleScheduler(learn, 4e-3, 30, div_factor=10, pct_end=0.1), TrueWD(learn, 0.3)]
learn.fit(30, 2e-3, wd=1e-4, callbacks=scheds)<jupyter_output><empty_output><jupyter_text>2e-3, 0.1, 76.4%1e-3, 0.1, 76.3%5e-4, 0.1, 76.5%4e-3, 0.1, 77.8%  With perspective wrap<jupyter_code>def find_coeffs(ori_pts, targ_pts):
    matrix = []
    for p1, p2 in zip(targ_pts, ori_pts):
        matrix.append([p1[0], p1[1], 1, 0, 0, 0, -p2[0]*p1[0], -p2[0]*p1[1]])
        matrix.append([0, 0, 0, p1[0], p1[1], 1, -p2[1]*p1[0], -p2[1]*p1[1]])

    A = FloatTensor(matrix)
    B = FloatTensor(ori_pts).view(8)
    #The 8 scalars we seek are solution of AX = B, we use the pseudo inverse to compute them.
    
    res = torch.mv(torch.mm(torch.inverse(torch.mm(A.t(),A)), A.t()), B)
    #res = numpy.dot(numpy.linalg.inv(A.T * A) * A.T, B)
    return res
def add_ones(coords):
    coords = coords.view(-1,2)
    ones = torch.ones(coords.size(0)).unsqueeze(1)
    coords = torch.cat([coords, ones], 1)
    return coords
def apply_perspective(coords, coeffs):
    ori_size = coords.size()
    #compress all the dims expect the last one ang adds ones, coords become N * 3
    coords = add_ones(coords)
    #Transform the coeffs in a 3*3 matrix with a 1 at the bottom left
    coeffs = torch.cat([coeffs, FloatTensor([1])]).view(3,3)
    coords = torch.mm(coords, coeffs.t())
    coords.mul_(1/coords[:,2].unsqueeze(1))
    return coords[:,:2].view(ori_size)
@reg_transform
def perspective_warp(c, img_size, magnitude:uniform=0) -> TfmType.Coord:
    magnitude = magnitude.view(4,2)
    ori_pts = [[-1,-1], [-1,1], [1,-1], [1,1]]
    targ_pts = [[x+m for x,m in zip(xs, ms)] for xs, ms in zip(ori_pts, magnitude)]
    coeffs = find_coeffs(ori_pts, targ_pts)
    return apply_perspective(c, coeffs)
def rand_int(low,high): return random.randint(low, high)
@reg_affine
def zoom(scale: uniform = 1.0, row_pct:uniform = 0.5, col_pct:uniform = 0.5) -> TfmType.Affine:
    s = 1-1/scale
    col_c = s * (2*col_pct - 1)
    row_c = s * (2*row_pct - 1)
    return [[1/scale, 0,       col_c],
            [0,       1/scale, row_c],
            [0,       0,       1.    ]]
@reg_transform
def tilt(c, img_size, direction:rand_int, magnitude:uniform=0) -> TfmType.Coord:
    ori_pts = [[-1,-1], [-1,1], [1,-1], [1,1]]
    if direction == 0:   targ_pts = [[-1,-1], [-1,1], [1,-1-magnitude], [1,1+magnitude]]
    elif direction == 1: targ_pts = [[-1,-1-magnitude], [-1,1+magnitude], [1,-1], [1,1]]
    elif direction == 2: targ_pts = [[-1,-1], [-1-magnitude,1], [1,-1], [1+magnitude,1]]
    elif direction == 3: targ_pts = [[-1-magnitude,-1], [-1,1], [1+magnitude,-1], [1,1]]  
    coeffs = find_coeffs(ori_pts, targ_pts)
    return apply_perspective(c, coeffs)
@reg_affine
def zoom1(scale: uniform = 1.0, row_pct:uniform = 0.5, col_pct:uniform = 0.5) -> TfmType.Affine:
    s = 1-math.sqrt(scale)
    col_c = s * (2*col_pct - 1)
    row_c = s * (2*row_pct - 1)
    return [[math.sqrt(scale), 0,       col_c],
            [0,       math.sqrt(scale), row_c],
            [0,       0,       1.    ]]
@reg_affine
def stretch(scale: uniform = 1.0) -> TfmType.Affine:
    return [[math.sqrt(scale), 0,       0],
            [0,       1/math.sqrt(scale), 0],
            [0,       0,       1.    ]]
sz = 224
trn_tfms = [stretch_tfm(scale=(0.75,1.33)),
            zoom_tfm(scale=(0.08,0.8), row_pct=(0,1.), col_pct=(0,1.)),
            flip_lr_tfm(p=0.5),
            center_crop_tfm(b=(0,1)),
            normalize_tfm(mean=data_mean,std=data_std)] #torchvision.transforms.RandomRotation(10),
val_tfms = [center_crop_tfm(b=(0,1)),
            normalize_tfm(mean=data_mean,std=data_std)]
#classes = ['airplanes','Motorbikes','Faces','watch','Leopards']
np.random.seed(42)
train_ds,valid_ds = ImageDataset.from_folder(PATH, test_pct=0.2)
classes = train_ds.classes
train_ds = TfmDataset(train_ds, trn_tfms, size=224)
valid_ds = TfmDataset(valid_ds, val_tfms, size=224)
x,y = train_ds[0]
x,y = valid_ds[0]
data = DataBunch(train_ds, valid_ds, bs=64, num_workers=8)
model = Darknet([1, 2, 4, 6, 3], num_classes=len(classes), nf=16).cuda()
learn = Learner(data, model)
learn.loss_fn = F.cross_entropy
learn.metrics = [accuracy]
learn.opt_fn = partial(optim.Adam, betas=(0.95,0.99))
scheds = [OneCycleScheduler(learn, 4e-3, 30, div_factor=10, pct_end=0.1), TrueWD(learn, 0.1)]
learn.fit(30, 2e-3, wd=1e-4, callbacks=scheds)
show_image_batch(data.train_dl, classes, rows=4)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1612443292287751, 'fraction_numerical': 0.03836681788723267, 'mean_word_length': 3.065331928345627, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '105666', 'license': 'Apache-2.0', 'n_tokens_mistral': 3179, 'n_tokens_neox': 3080, 'n_words': 621, 'path': 'deep-learning/fastai-docs/fastai_docs-master/dev_nb/experiments/caltech101.ipynb', 'repo_name': 'AadityaGupta/Artificial-Intelligence-Deep-Learning-Machine-Learning-Tutorials', 'size': '7715'}