<jupyter_start><jupyter_text>Creating a tab+ NLP model  load previous models   NLP model<jupyter_code>learn_nlp = load_learner('.','final_nlp_word.pkl')
learn_nlp.model[-1].layers = learn_nlp.model[-1].layers[:-3] 
learn_nlp.model

flatten_model(learn_nlp.model)<jupyter_output><empty_output><jupyter_text>Load tabular learner<jupyter_code>learn_tab = load_learner('.','final_tab.pkl')
learn_tab<jupyter_output><empty_output><jupyter_text>Grab up to ReLU<jupyter_code>learn_tab.model.layers = learn_tab.model.layers[:-3]
learn_tab.model<jupyter_output><empty_output><jupyter_text>Concat dataset<jupyter_code>class ConcatDataset(Dataset):
    def __init__(self, x1, x2, y): 
        self.x1,self.x2,self.y = x1,x2,y
    def __len__(self): 
        return len(self.y)
    def __getitem__(self, i): 
        return (self.x1[i], self.x2[i]), self.y[i]<jupyter_output><empty_output><jupyter_text>Load datasets<jupyter_code>data_tab = load_data('.','tab_data')
data_nlp = load_data('.', 'nlp_data')<jupyter_output><empty_output><jupyter_text>Create ConcatDatasets for tab and NLP<jupyter_code>train_ds = ConcatDataset(data_tab.train_ds.x, data_nlp.train_ds.x, data_tab.train_ds.y)
valid_ds = ConcatDataset(data_tab.valid_ds.x, data_nlp.valid_ds.x, data_tab.valid_ds.y)
# 
def collate(batch):    
    x,y = list(zip(*batch))
    x1,x2 = list(zip(*x))
    x1 = to_data(x1)
    x1 = list(zip(*x1))
    x1 = torch.stack(x1[0]), torch.stack(x1[1])
    x2, y = pad_collate(list(zip(x2, y)), pad_idx=1, pad_first=True)
    return (x1, x2), y
# Go through the text data by order of length with a bit of randomness.
train_sampler = SortishSampler(data_nlp.train_ds.x, key=lambda t: len(data_nlp.train_ds[t][0].data), bs=bs//2)

# Go through the text data by order of length.
valid_sampler = SortSampler(data_nlp.valid_ds.x, key=lambda t: len(data_nlp.valid_ds[t][0].data))
train_dl = DataLoader(train_ds, bs, sampler=train_sampler)
valid_dl = DataLoader(valid_ds, bs, sampler=valid_sampler)
data = DataBunch(train_dl, valid_dl, device=defaults.device, collate_fn=collate, path='.')
class ConcatModel(nn.Module):
    def __init__(self, model_tab, model_nlp, layers, drops): 
        super().__init__()
        self.model_tab = model_tab
        self.model_nlp = model_nlp
        lst_layers = []
        activs = [nn.ReLU(inplace=True),] * (len(layers)-2) + [None]
        for n_in,n_out,p,actn in zip(layers[:-1], layers[1:], drops, activs): 
            lst_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn) # https://docs.fast.ai/layers.html#bn_drop_lin
        self.layers = nn.Sequential(*lst_layers)

    def forward(self, *x):
        x_tab = self.model_tab(*x[0])
        x_nlp = self.model_nlp(x[1])[0]
        x = torch.cat([x_tab, x_nlp], dim=1)
        return self.layers(x)<jupyter_output><empty_output><jupyter_text>Layer groups Layer groups allow us to use the fastai freeze layers approach to train some layers more or turn off training on some layers. ```flatten_model``` allows you to acccess the model as an array making it easy to grab the bits you want<jupyter_code>m = simple_cnn([3,6,12])
m
m[0]
print(flatten_model(m)[0])
print('-*-')
print(flatten_model(m)[1])

lin_layers = [432+50, 24]
ps = [0.8]
model = ConcatModel(learn_tab.model, learn_nlp.model, lin_layers, ps)
model
layer_groups = [nn.Sequential(*flatten_model(learn_nlp.layer_groups[0][0])),
                nn.Sequential(*flatten_model(learn_nlp.layer_groups[0][1])),
                nn.Sequential(*flatten_model(learn_nlp.layer_groups[0][2])),
                nn.Sequential(*flatten_model(learn_nlp.layer_groups[0][3])),
                nn.Sequential(*(flatten_model(learn_nlp.layer_groups[0][4]) + 
                                flatten_model(model.model_tab) +
                                flatten_model(model.layers)))] 
loss_func = nn.CrossEntropyLoss()
f1 = FBeta(average='macro')
learn = Learner(data, model, loss_func=loss_func, metrics=[accuracy, f1],layer_groups=layer_groups)
learn.layer_groups
learn.freeze()
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(20,max_lr=1e-02,
                   callbacks=[SaveModelCallback(learn, 
                                                monitor='f_beta',
                                                    every='improvement',
                                                     name='best_concat')])
learn.recorder.plot()
learn.load('best_concat')<jupyter_output><empty_output><jupyter_text>Unfreeze some layers ❄️<jupyter_code>learn.freeze_to(-2) 
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(15, max_lr=slice(1e-06,1e-03),
                   callbacks=[SaveModelCallback(learn, 
                                                monitor='f_beta',
                                                    every='improvement',
                                                     name='best_concat[-2]')])
learn.load('best_concat[-2]');
learn.freeze_to(-3)
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(20, max_lr=slice(1e-06,1e-03),
                     callbacks=[SaveModelCallback(learn, 
                                                monitor='f_beta',
                                                    every='improvement',
                                                     name='best_concat[-3]')])
learn.load('best_concat[-3]');
learn.freeze_to(-3)
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(40, max_lr=slice(1e-06,1e-03),
                    callbacks=[SaveModelCallback(learn, 
                                                monitor='f_beta',
                                                    every='improvement',
                                                     name='best_concat[-3b]')])
learn.load('best_concat[-3b]');
learn.lr_find()
learn.recorder.plot(suggestion=True)
learn.fit_one_cycle(40, max_lr=1.00E-05,
                                       callbacks=[SaveModelCallback(learn,
                                                monitor='f_beta',
                                                    every='improvement',
                                                     name='best_concat[-4]')])(20, max_lr=slice(1e-06,1e-02))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13317112733171127, 'fraction_numerical': 0.019464720194647202, 'mean_word_length': 2.610070257611241, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '384111', 'license': 'MIT', 'n_tokens_mistral': 2128, 'n_tokens_neox': 2067, 'n_words': 308, 'path': 'notebooks/05_model.ipynb', 'repo_name': 'davanstrien/Website-Classification', 'size': '6165'}