<jupyter_start><jupyter_text>Prepare for modeling  Get files<jupyter_code>#export
def compose(x, funcs, *args, order_key='_order', **kwargs):
    key = lambda o: getattr(o, order_key, 0)
    for f in sorted(listify(funcs), key=key): x = f(x, **kwargs)
    return x
ListContainer??
#export
class ItemList(ListContainer):
    def __init__(self, items, path='.', tfms=None):
        super().__init__(items)
        self.path, self.tfms = path, tfms
        
    def __repr__(self): return f'{super().__repr__()}\nPath: {self.path}'
    
    def new(self, items, cls=None):
        if cls is None: cls = self.__class__
        return cls(items, self.path, self.tfms)
    
    def get(self, i): return i
    def _get(self, i): return compose(self.get(i), self.tfms)
    
    def __getitem__(self, idx):
        res = super().__getitem__(idx)
        if isinstance(res, list): return [self._get(o) for o in res]
        return self._get(res)
#export
class ImageList(ItemList):
    @classmethod
    def from_files(cls, path, extensions=None, recurse=True, include=None, **kwargs):
        if extensions is None: extensions = image_extensions
        return cls(get_files(path, extensions, recurse, include), path, **kwargs)
    
    def get(self, fn): return PIL.Image.open(fn)
#export
class Transform(): _order = 0
    
class MakeRGB(Transform):
    def __call__(self, item): return item.convert('RGB')
    
def make_rgb(item): return item.convert('RGB')
il = ImageList.from_files(path, tfms=make_rgb)
il
img = il[0]; img
il[:1]<jupyter_output><empty_output><jupyter_text>Split validation set<jupyter_code>fn = il.items[0]; fn
fn.parent.parent.name
#export
def grandparent_splitter(fn, train_name='train', valid_name='valid'):
    gp = fn.parent.parent.name
    return True if gp == valid_name else False if gp == train_name else None

def split_by_func(items, f):
    mask = [f(o) for o in items]
    
    t = [o for o,m in zip(items, mask) if m == False]
    v = [o for o,m in zip(items, mask) if m == True]
    
    return t,v
splitter = partial(grandparent_splitter, valid_name='val')
%time train, valid = split_by_func(il, splitter)
len(train), len(valid)
#export
class SplitData():
    def __init__(self, train, valid): self.train, self.valid = train, valid
        
    def __getattr__(self, k): return getattr(self.train, k)
    
    def __setstate__(self, data:Any): self.__dict__.update(data)
        
    @classmethod
    def split_by_func(cls, il, f):
        lists = map(il.new, split_by_func(il.items, f))
        return cls(*lists)
    
    def __repr__(self): return f'{self.__class__.__name__}\Train: {self.train}\nValid: {self.valid}'
sd = SplitData.split_by_func(il, splitter); sd<jupyter_output><empty_output><jupyter_text>Labeling<jupyter_code>#export
from collections import OrderedDict

def uniqueify(x, sort=False):
    res = list(OrderedDict.fromkeys(x).keys())
    if sort: res.sort()
    return res
#export
class Processor():
    def process(self, items): return items
    
class CategoryProcessor(Processor):
    def __init__(self): self.vocab = None
        
    def __call__(self, items):
        if self.vocab is None:
            self.vocab = uniqueify(items)
            self.otoi = {v:k for k,v in enumerate(self.vocab)}
        return [self.proc1(o) for o in items]
    
    def proc1(self, item): return self.otoi[item]
    
    def deprocess(self, idxs):
        assert self.vocab is not None
        return [self.deproc1(idx) for idx in idxs]
    
    def deproc1(self, idx): return self.vocab[idx]
#export
def parent_labeler(fn): return fn.parent.name

def _label_by_func(il, f, cls=ItemList): return cls([f(o) for o in il.items], path=il.path)
#export
class LabeledData():
    def process(self, il, proc): return il.new(compose(il.items, proc))
    
    def __init__(self, x, y, proc_x=None, proc_y=None):
        self.x, self.y = self.process(x, proc_x), self.process(y, proc_y)
        self.proc_x, self.proc_y = proc_x, proc_y
        
    def __repr__(self): return f'{self.__class__.__name__}\nx: {self.x}\ny: {self.y}\n'
    def __getitem__(self, idx): return self.x[idx], self.y[idx]
    def __len__(self): return len(self.x)
    
    def x_obj(self, idx): return self.obj(self.x, idx, self.proc_x)
    def y_obj(self, idx): return self.obj(self.y, idx, self.proc_y)
    
    def obj(self, items, idx, procs):
        isint = isinstance(idx, int) or (isinstance(idx, torch.LongTensor) and not idx.ndim)
        item = items[idx]
        for proc in reversed(listify(procs)):
            item = proc.deproc1(item) if isint else proc.deprocess(item)
        return item
    
    @classmethod
    def label_by_func(cls, il, f, proc_x=None, proc_y=None):
        return cls(il, _label_by_func(il, f), proc_x, proc_y)
    
def label_by_func(sd, f, proc_x=None, proc_y=None):
    train = LabeledData.label_by_func(sd.train, f, proc_x, proc_y)
    valid = LabeledData.label_by_func(sd.valid, f, proc_x, proc_y)
    return SplitData(train, valid)            
ll = label_by_func(sd, parent_labeler, proc_y=CategoryProcessor())
ll
assert ll.train.proc_y is ll.valid.proc_y
ll.train.y
ll.train.y.items[0], ll.train.y_obj(0), ll.train.y_obj(range(2))<jupyter_output><empty_output><jupyter_text>Transform to tensor<jupyter_code>ll.train[0]
ll.train[0][0]
ll.train[0][0].resize((128,128))
#export
class ResizeFixed(Transform):
    _order = 10
    def __init__(self, size):
        if isinstance(size, int): size = (size,size)
        self.size = size
        
    def __call__(self, item): return item.resize(self.size, PIL.Image.BILINEAR)
    
def to_byte_tensor(item):
    res = torch.ByteTensor(torch.ByteStorage.from_buffer(item.tobytes()))
    w, h = item.size
    return res.view(h,w,-1).permute(2,0,1)
to_byte_tensor._order = 20

def to_float_tensor(item): return item.float().div_(255.)
to_float_tensor._order = 30
tfms = [make_rgb, ResizeFixed(128), to_byte_tensor, to_float_tensor]

il = ImageList.from_files(path, tfms=tfms)
sd = SplitData.split_by_func(il, splitter)
ll = label_by_func(sd, parent_labeler, proc_y=CategoryProcessor())
#export
def show_image(im, figsize=(3,3)):
    plt.figure(figsize=figsize)
    plt.axis('off')
    plt.imshow(im.permute(1,2,0))
x, y = ll.train[0]
x.shape
show_image(x)<jupyter_output><empty_output><jupyter_text>Modeling<jupyter_code>bs = 64
train_dl, valid_dl = get_dls(ll.train, ll.valid, bs, num_workers=4)
x,y = next(iter(train_dl))
x.shape
show_image(x[0])
ll.train.proc_y.vocab[y[0]]
y
#export
class DataBunch():
    def __init__(self, train_dl, valid_dl, c_in=None, c_out=None):
        self.train_dl, self.valid_dl, self.c_in, self.c_out = train_dl, valid_dl, c_in, c_out
        
    @property
    def train_ds(self): return self.train_dl.dataset
    @property
    def valid_ds(self): return self.valid_dl.dataset
#export
def databunchify(sd, bs, c_in=None, c_out=None, **kwargs):
    dls = get_dls(sd.train, sd.valid, bs, **kwargs)
    return DataBunch(*dls, c_in=c_in, c_out=c_out)

SplitData.to_databunch = databunchify
path = untar_data(URLs.IMAGENETTE_160)
tfms = [make_rgb, ResizeFixed(128), to_byte_tensor, to_float_tensor]

il = ImageList.from_files(path, tfms=tfms)
sd = SplitData.split_by_func(il, partial(grandparent_splitter, valid_name='val'))
ll = label_by_func(sd, parent_labeler, proc_y=CategoryProcessor())
data = ll.to_databunch(bs, c_in=3, c_out=10, num_workers=4)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>cbfs = [CudaCallback, Recorder,
      partial(AvgStatsCallback, accuracy)]
m, s = x.mean((0,2,3)).cuda(), x.std((0,2,3)).cuda()
m, s
#export
def normalize_chan(x, mean, std):
    return (x - mean[...,None,None]) / std[...,None,None]

_m = tensor([0.47, 0.48, 0.45])
_s = tensor([0.29, 0.28, 0.30])
norm_imagenette = partial(normalize_chan, mean=_m.cuda(), std=_s.cuda())
cbfs.append(partial(BatchTransformXCallback, norm_imagenette))
nfs = [64,64,128,256]
#export
import math
def prev_pow_2(x): return 2**math.floor(math.log2(x))
#export
def get_cnn_layers(data, nfs, layer, **kwargs):
    def f(ni, nf, stride=2): return layer(ni, nf, 3, stride=stride, **kwargs)
    l1 = data.c_in
    l2 = prev_pow_2(l1*3*3)
    layers = [f(l1, l2, stride=1),
             f(l2, l2*2, stride=2),
             f(l2*2, l2*4, stride=2)]
    nfs = [l2*4] + nfs
    
    layers += [f(nfs[i], nfs[i+1]) for i in range(len(nfs) - 1)]
    layers += [nn.AdaptiveAvgPool2d(1), Lambda(flatten), nn.Linear(nfs[-1], data.c_out)]
    
    return layers

def get_cnn_model(data, nfs, layer, **kwargs):
    return nn.Sequential(*get_cnn_layers(data, nfs, layer, **kwargs))

def get_learn_run(nfs, data, lr, layer, cbs=None, opt_func=None, **kwargs):
    model = get_cnn_model(data, nfs, layer, **kwargs)
    init_cnn(model)
    return get_runner(model, data, lr=lr, cbs=cbs, opt_func=opt_func)
sched = combine_scheds([0.3, 0.7], cos_1cycle_anneal(0.1, 0.3, 0.05))
learn, run = get_learn_run(nfs, data, 0.2, conv_layer, cbs=cbfs+[partial(ParamScheduler, 'lr', sched)])
#export
def model_summary(run, learn, data, find_all=False):
    xb, yb = get_batch(data.valid_dl, run)
    device = next(learn.model.parameters()).device
    xb, yb, = xb.to(device), yb.to(device)
    mods = find_modules(learn.model, is_lin_layer) if find_all else learn.model.children()
    f =  lambda hook,mod,inp,outp: print(f'{mod}\n{outp.shape}\n')
    with Hooks(mods, f) as hooks: learn.model(xb)
model_summary(run, learn, data)
%time run.fit(5, learn)<jupyter_output>train: [1.7865281197196115, tensor(0.3803, device='cuda:0')]
valid: [1.6788043640525478, tensor(0.4346, device='cuda:0')]
train: [1.4219498742607455, tensor(0.5268, device='cuda:0')]
valid: [1.3536269655652866, tensor(0.5483, device='cuda:0')]
train: [1.0226649337641252, tensor(0.6621, device='cuda:0')]
valid: [1.2033512888136944, tensor(0.6125, device='cuda:0')]
train: [0.6602305570314975, tensor(0.7879, device='cuda:0')]
valid: [1.1022392515923567, tensor(0.6499, device='cuda:0')]
train: [0.37651648739472227, tensor(0.8951, device='cuda:0')]
valid: [1.094550283638535, tensor(0.6586, device='cuda:0')]
CPU times: user 43.3 s, sys: 20.5 s, total: 1min 3s
Wall time: 1min 25s
{'fraction_non_alphanumeric': 0.14982716049382716, 'fraction_numerical': 0.03762962962962963, 'mean_word_length': 4.208847736625515, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '5961', 'license': 'Apache-2.0', 'n_tokens_mistral': 4108, 'n_tokens_neox': 3927, 'n_words': 842, 'path': 'nbs/dl2/selfmade/08_data_block_selfmade.ipynb', 'repo_name': 'Hustens0hn/fastai-course-v3', 'size': '10125'}