<jupyter_start><jupyter_text>hyperpar scan<jupyter_code>def default_cfg():
    return Config(
        sites=['SSW'], 
        use_neptune=True, 
        n_epochs=30, 
        bs=32, 
        lr=1e-3,
        model='resnest50-hyper-scan',
        scheduler='torch.optim.lr_scheduler.CosineAnnealingLR',
        loss_fn='lark.ops.FocalLoss',
        use_pink_noise=0,
        use_recorded_noise=False,
        use_overlays=False,
        apply_filter=0,
    )


par_dict = dict(
    use_pink_noise=0.5,
    use_recorded_noise=0.5,
    use_overlays=True,
    apply_filter=0.5,    
)

def apply_par(k):
    cfg = default_cfg()
    setattr(cfg, k, par_dict[k])
    return cfg

configs = [apply_par(k) for k in par_dict]
    
def make_model():
    prep = MixedSig2Spec(cfg)
    main_model = torch.hub.load('zhanghang1989/ResNeSt', 'resnest50', pretrained=True)
    posp = torch.nn.Sequential(
        torch.nn.Linear(in_features=2048, out_features=1024, bias=True),
        torch.nn.Dropout(p=0.2),
        torch.nn.ReLU(),
        torch.nn.Linear(in_features=1024, out_features=512, bias=True),
        torch.nn.Dropout(p=0.2),
        torch.nn.ReLU(),
        torch.nn.Linear(in_features=512, out_features=len(cfg.labels), bias=True),
    )
    main_model.fc = posp
    model = torch.nn.Sequential(prep, main_model)
    model = model.cuda()
    return model

for cfg in configs:
    model = make_model()
    lrn = Learner("resnest50-param-scan", cfg, model)
    lrn.learn()<jupyter_output>resnest50-focal-loss-20210521-193723
{'fraction_non_alphanumeric': 0.13083497698882315, 'fraction_numerical': 0.04142011834319527, 'mean_word_length': 2.9125964010282774, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '599857', 'license': 'Apache-2.0', 'n_tokens_mistral': 631, 'n_tokens_neox': 581, 'n_words': 81, 'path': 'nbs/botkop/018-focal-loss.ipynb', 'repo_name': 'botkop/lark', 'size': '1521'}