<jupyter_start><jupyter_text>Get data<jupyter_code>def unpack_sample(feats):
    return {
        'class' : unpack_int64_list(feats['class']),
        'image' : unpack_bytes_list(feats['image'])
    }

train_fnames = get_files(ROOT/'data/flowers/train', extensions='.tfrec')
test_fnames = get_files(ROOT/'data/flowers/val', extensions='.tfrec')

data = []
for name in train_fnames+test_fnames:
    r = Reader(str(name), unpack_sample)
    for sample in r:
        data.append([sample['image'][0], sample['class'][0]])
get_x = lambda o: PILImage.create(io.BytesIO(o[0]))
get_y = lambda o: o[1]

def get_items(noop): return data

splitter = IndexSplitter(range(12753, len(data)))
item_tfms = [RandomResizedCrop(224, min_scale=0.7)]
batch_tfms = [*aug_transforms()]

block = DataBlock(
    blocks=(ImageBlock, CategoryBlock), 
    get_items=get_items,
    get_x=get_x,
    get_y=get_y,
    splitter=splitter,
    item_tfms=item_tfms,
    batch_tfms=batch_tfms
)

dls = block.dataloaders('', bs=64)<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>model = xresnet50_deconv(n_out=dls.c)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy)

learn.lr_find()<jupyter_output><empty_output><jupyter_text>lr=6e-3, wd=1e-2<jupyter_code>model = xresnet50_deconv(n_out=dls.c)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 6e-3, wd=1e-2)<jupyter_output><empty_output><jupyter_text>+ self-attention<jupyter_code>model = xresnet50_deconv(n_out=dls.c, sa=True)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 6e-3, wd=1e-2)<jupyter_output><empty_output><jupyter_text>+ Mish<jupyter_code>model = xresnet50_deconv(n_out=dls.c, sa=True, act_cls=MishCuda)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy)
learn.fit_flat_cos(5, 6e-3, wd=1e-2)<jupyter_output><empty_output><jupyter_text>+ LabelSmoothingCrossEntropy<jupyter_code>model = xresnet50_deconv(n_out=dls.c, sa=True, act_cls=MishCuda)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy())
learn.fit_flat_cos(5, 6e-3, wd=1e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>+ maxBlurPool<jupyter_code>model = xresnet50_deconv(n_out=dls.c, sa=True, act_cls=MishCuda)
model = replace_model_layer(model, nn.MaxPool2d, kornia.contrib.MaxBlurPool2d(3, True))
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy())
learn.fit_flat_cos(5, 6e-3, wd=1e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>+ increase wd<jupyter_code>model = xresnet50_deconv(n_out=dls.c, sa=True, act_cls=MishCuda)
model = replace_model_layer(model, nn.MaxPool2d, kornia.contrib.MaxBlurPool2d(3, True))
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy())
learn.fit_flat_cos(5, 6e-3, wd=5e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>+ earlier annealing<jupyter_code>model = xresnet50_deconv(n_out=dls.c, sa=True, act_cls=MishCuda)
model = replace_model_layer(model, nn.MaxPool2d, kornia.contrib.MaxBlurPool2d(3, True))
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy())
learn.fit_flat_cos(5, 6e-3, pct_start=0.5, wd=5e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>+ early annealing -maxblurpool<jupyter_code>model = xresnet50_deconv(n_out=dls.c, sa=True, act_cls=MishCuda)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy())
learn.fit_flat_cos(5, 6e-3, pct_start=0.5, wd=5e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>+ early annealing -maxblurpool, original wd<jupyter_code>model = xresnet50_deconv(n_out=dls.c, sa=True, act_cls=MishCuda)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy())
learn.fit_flat_cos(5, 6e-3, pct_start=0.5, wd=1e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>+ lowered label smoothing, + slightly increased wd<jupyter_code>model = xresnet50_deconv(n_out=dls.c, sa=True, act_cls=MishCuda)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
learn.fit_flat_cos(5, 6e-3, wd=3e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>Best hyperparams:  - lr=6e-3  - wd=3e-2?  - sa=True  - act_cls=MishCuda  - loss_func=LabelSmoothingCrossEntropy() -- perhaps eps=0.05?  - fit_flat_cos with default annealing  tweaked xse_resnext9_deconv see e.g., `mini_net` from https://github.com/pete88b/data-science/blob/master/fastai-things/train-imagenette-mininet.ipynb<jupyter_code>def xse_resnext9_deconv(pretrained=False, **kwargs):
    return XResNet_deconv(
        SEResNeXtBlock, 
        expansion=1, 
        layers=[1,1,1,1], 
        **kwargs
    )
model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=32, reduction=16)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
# xse_resnext9 with deconv stem
model<jupyter_output><empty_output><jupyter_text>Test default groups/reduction = 32/16<jupyter_code>model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=32, reduction=16)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
learn.fit_flat_cos(5, 6e-3, wd=3e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>try groups/reduction = 64/8<jupyter_code>model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
learn.fit_flat_cos(5, 6e-3, wd=3e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>Increase lr<jupyter_code>model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
learn.fit_flat_cos(5, 1e-2, wd=3e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>Re-up labelsmoothing eps<jupyter_code>model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.1))
learn.fit_flat_cos(5, 1e-2, wd=3e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>decrease wd=1e-2<jupyter_code>model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
learn.fit_flat_cos(5, 1e-2, wd=1e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>decrease wd=6e-3<jupyter_code>model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
learn.fit_flat_cos(5, 1e-2, wd=6e-3)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>increase lr=3e-2<jupyter_code>model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
learn.fit_flat_cos(5, 3e-2, wd=1e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>+ maxblurpool<jupyter_code>model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
model = replace_model_layer(model, nn.MaxPool2d, kornia.contrib.MaxBlurPool2d(3, True))
learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
learn.fit_flat_cos(5, 1e-2, wd=1e-2)<jupyter_output>@log_args had an issue on LabelSmoothingCrossEntropy.__init__ -> missing a required argument: 'self'<jupyter_text>Final xse_resnext9 with deconv results:   With MBP: 77.76% ± 0.28%<jupyter_code>for i in range(5):
    model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
    model = replace_model_layer(model, nn.MaxPool2d, kornia.contrib.MaxBlurPool2d(3, True))
    learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
    learn.fit_flat_cos(5, 1e-2, wd=1e-2)
res = np.array([0.778825, 0.779903, 0.773168, 0.780711, 0.775593])
print('{:.2f}% +/- {:.2f}%'.format(100*res.mean(), 100*res.std()))<jupyter_output>77.76% +/- 0.28%<jupyter_text>Increased wd: 77.75% ± 0.38%<jupyter_code>for i in range(5):
    model = xse_resnext9_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
    model = replace_model_layer(model, nn.MaxPool2d, kornia.contrib.MaxBlurPool2d(3, True))
    learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
    learn.fit_flat_cos(5, 1e-2, wd=3e-2)
res = np.array([0.780981, 0.782328, 0.772090, 0.777209, 0.775054])
print('{:.2f}% +/- {:.2f}%'.format(100*res.mean(), 100*res.std()))<jupyter_output>77.75% +/- 0.38%<jupyter_text>Larger model (xse_resnext18): 79.53% ± 0.50%<jupyter_code>def xse_resnext18_deconv(pretrained=False, **kwargs):
    return XResNet_deconv(
        SEResNeXtBlock, 
        expansion=1, 
        layers=[2, 2, 2, 2], 
        **kwargs
    )
for i in range(5):
    model = xse_resnext18_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
    model = replace_model_layer(model, nn.MaxPool2d, kornia.contrib.MaxBlurPool2d(3, True))
    learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
    learn.fit_flat_cos(5, 1e-2, wd=1e-2)
res = np.array([0.804688, 0.790948, 0.795528, 0.794181, 0.791218])
print('{:.2f}% +/- {:.2f}%'.format(100*res.mean(), 100*res.std()))<jupyter_output>79.53% +/- 0.50%<jupyter_text>And even larger (xse_resnext34): 79.92% ± 0.72%<jupyter_code>def xse_resnext34_deconv(pretrained=False, **kwargs):
    return XResNet_deconv(
        SEResNeXtBlock, 
        expansion=1, 
        layers=[3, 4, 6, 3], 
        **kwargs
    )
for i in range(5):
    model = xse_resnext34_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
    model = replace_model_layer(model, nn.MaxPool2d, kornia.contrib.MaxBlurPool2d(3, True))
    learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05))
    learn.fit_flat_cos(5, 1e-2, wd=1e-2)
res = np.array([0.812231, 0.794450, 0.795797, 0.792026, 0.801455])
print('{:.2f}% +/- {:.2f}%'.format(100*res.mean(), 100*res.std()))<jupyter_output>79.92% +/- 0.72%<jupyter_text>Any larger requires fp16/lower lr (xse_resnext50): 79.95% ± 0.23%<jupyter_code>def xse_resnext50_deconv(pretrained=False, **kwargs):
    return XResNet_deconv(
        SEResNeXtBlock, 
        expansion=4, 
        layers=[3, 4, 6, 3], 
        **kwargs
    )
for i in range(5):
    model = xse_resnext50_deconv(n_out=dls.c, sa=True, act_cls=MishCuda, groups=64, reduction=8)
    model = replace_model_layer(model, nn.MaxPool2d, kornia.contrib.MaxBlurPool2d(3, True))
    learn = Learner(dls, model, opt_func=ranger, metrics=accuracy, loss_func=LabelSmoothingCrossEntropy(eps=0.05)).to_fp16()
    learn.fit_flat_cos(5, 6e-3, wd=1e-2)
res = np.array([0.803071, 0.801185, 0.796875, 0.797953, 0.798222])
print('{:.2f}% +/- {:.2f}%'.format(100*res.mean(), 100*res.std()))<jupyter_output>79.95% +/- 0.23%
{'fraction_non_alphanumeric': 0.1416473595909563, 'fraction_numerical': 0.0518494847008069, 'mean_word_length': 7.034659820282413, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 100, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '26', 'dir': 'jupyter-structured-clean-dedup', 'id': '513229', 'license': 'MIT', 'n_tokens_mistral': 5549, 'n_tokens_neox': 5198, 'n_words': 780, 'path': 'nbs/03. Network deconvolution experiments.ipynb', 'repo_name': 'jwuphysics/FastGarden', 'size': '12517'}