<jupyter_start><jupyter_text>Twitter - kaggle -  ULMFit<jupyter_code>%reload_ext autoreload
%autoreload 2
%matplotlib inline

from fastai import *
from fastai.text import *
from utils import *
from sklearn.model_selection import KFold
torch.cuda.is_available()
from pathlib import PosixPath
path = PosixPath('/home/shanmugam/fastai/')
# path = PosixPath('/home/shanmugam/.fastai/models/wt103-fwd')
lm_fns = ['learn_en_wiki_15000', 'learn_en_wiki_15_vocab']
path

train_df = pd.read_csv(path/'train.csv')
train_df.loc[pd.isna(train_df.text),'text']='NA'
train_df.head()
test_df = pd.read_csv(path/'test.csv')
test_df.loc[pd.isna(test_df.text),'text']='NA'
test_df.head()
df = pd.concat([train_df,test_df], sort=False)
bs=128
data_lm = (TextList.from_df(df, path, cols='text')
    .split_by_rand_pct(0.1, seed=42)
    .label_for_lm()           
    .databunch(bs=bs, num_workers=1))
#learn_lm = language_model_learner(data_lm, AWD_LSTM, drop_mult=1.0)
learn_lm = language_model_learner(data_lm, AWD_LSTM, pretrained_fnames=lm_fns, drop_mult=1.0).to_fp16()
lr = 1e-3
lr *= bs/48
learn_lm.fit_one_cycle(2, lr*10, moms=(0.8,0.7))
learn_lm.unfreeze()
learn_lm.fit_one_cycle(8, lr, moms=(0.8,0.7))
lang='en'
learn_lm.save(f'{lang}fine_tuned')
learn_lm.save_encoder(f'{lang}fine_tuned_enc')<jupyter_output><empty_output><jupyter_text>Classifier<jupyter_code>data_clas = (TextList.from_df(train_df, path, vocab=data_lm.vocab, cols='text')
    .split_by_rand_pct(0.1, seed=42)
    .label_from_df(cols='target')
    .databunch(bs=bs, num_workers=1))

data_clas.save(f'{lang}_textlist_class')
data_clas = load_data(path, f'{lang}_textlist_class', bs=bs, num_workers=1)
from sklearn.metrics import f1_score
@np_func
def f1(inp,targ): return f1_score(targ, np.argmax(inp, axis=-1))
learn_c = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5, metrics=[accuracy,f1]).to_fp16()
learn_c.load_encoder(f'{lang}fine_tuned_enc')
learn_c.freeze()
lr=2e-2
lr *= bs/48
learn_c.fit_one_cycle(2, lr, moms=(0.8,0.7))
learn_c.fit_one_cycle(2, lr, moms=(0.8,0.7))
learn_c.freeze_to(-2)
learn_c.fit_one_cycle(2, slice(lr/(2.6**4),lr), moms=(0.8,0.7))
learn_c.freeze_to(-3)
learn_c.fit_one_cycle(2, slice(lr/2/(2.6**4),lr/2), moms=(0.8,0.7))
learn_c.unfreeze()
learn_c.fit_one_cycle(3, slice(lr/10/(2.6**4),lr/10), moms=(0.8,0.7))
learn_c.save(f'{lang}clas')
learn_lm.predict("John went to a football match", n_words=50)<jupyter_output><empty_output><jupyter_text>Backward<jupyter_code>bs=128
data_lm = (TextList.from_df(df, path, cols='text')
    .split_by_rand_pct(0.1, seed=42)
    .label_for_lm()           
    .databunch(bs=bs, num_workers=1, backwards=True))
#learn_lm = language_model_learner(data_lm, AWD_LSTM, drop_mult=1.0)
learn_lm = language_model_learner(data_lm, AWD_LSTM, config={**awd_lstm_lm_config, 'n_hid': 1152},
                                  pretrained_fnames=lm_fns, drop_mult=1.0)


lr = 1e-3
lr *= bs/48

learn_lm.fit_one_cycle(2, lr*10, moms=(0.8,0.7))


learn_lm.unfreeze()
learn_lm.fit_one_cycle(8, lr, moms=(0.8,0.7))
lang='en'
learn_lm.save(f'{lang}fine_tuned_bwd')
learn_lm.save_encoder(f'{lang}fine_tuned_enc_bwd')
data_clas = (TextList.from_df(train_df, path, vocab=data_lm.vocab, cols='text')
    .split_by_rand_pct(0.1, seed=42)
    .label_from_df(cols='target')
    .databunch(bs=bs, num_workers=1, backwards=True))

data_clas.save(f'{lang}_textlist_class_bwd')

data_clas = load_data(path, f'{lang}_textlist_class_bwd', bs=bs, num_workers=1, backwards=True)
learn_c = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5, metrics=[accuracy,f1]).to_fp16()
learn_c.load_encoder(f'{lang}fine_tuned_enc_bwd')
learn_c.freeze()
lr=2e-2
lr *= bs/48
learn_c.fit_one_cycle(2, lr, moms=(0.8,0.7))
learn_c.freeze_to(-2)
learn_c.fit_one_cycle(2, slice(lr/(2.6**4),lr), moms=(0.8,0.7))
learn_c.freeze_to(-3)
learn_c.fit_one_cycle(2, slice(lr/2/(2.6**4),lr/2), moms=(0.8,0.7))
learn_c.unfreeze()
learn_c.fit_one_cycle(3, slice(lr/10/(2.6**4),lr/10), moms=(0.8,0.7))
learn_c.save(f'{lang}clas_bwd')<jupyter_output><empty_output><jupyter_text>Ensemble<jupyter_code>data_clas = load_data(path, f'{lang}_textlist_class', bs=bs, num_workers=1)
data_clas.add_test(test_df)
learn_c = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5, metrics=[accuracy,f1]).to_fp16()
learn_c.load(f'{lang}clas', purge=False);
preds,targs = learn_c.get_preds(DatasetType.Test)
accuracy(preds,targs),f1(preds,targs)
data_clas_bwd = load_data(path, f'{lang}_textlist_class_bwd', bs=bs, num_workers=1, backwards=True)
data_clas_bwd.add_test(test_df)
learn_c_bwd = text_classifier_learner(data_clas_bwd, AWD_LSTM, drop_mult=0.5, metrics=[accuracy,f1]).to_fp16()
learn_c_bwd.load(f'{lang}clas_bwd', purge=False);
preds_b,targs_b = learn_c_bwd.get_preds(DatasetType.Test)
accuracy(preds_b,targs_b),f1(preds_b,targs_b)
DatasetType.Test
preds_avg = (preds+preds_b)/2
accuracy(preds_avg,targs_b),f1(preds_avg,targs_b)
sub = pd.read_csv(path/'sample_submission.csv')
sub.target = preds_avg.argmax(dim=-1)
sub
sub['target'].value_counts()
sub.to_csv('model_lin.csv', index=False)<jupyter_output><empty_output><jupyter_text>MixedTabular ItemList<jupyter_code>train_df.head()

cat_cols = [
    'keyword', 
    'location', 
]

# cont_cols = [
#     'c', 
#     'funny',
#     'useful',
#     'user_average_stars',
#     'business_stars'
# ]
cont_cols = None
txt_cols = ['text']

dep_var = ['target']
procs = [FillMissing, Categorify]
data_cls_mx = (MixedTabularList.from_df(train_df, cat_cols, cont_cols, txt_cols,
                                     vocab=data_lm.vocab, procs=procs, path=path)
          .split_by_rand_pct(valid_pct=0.1, seed=42)
          .label_from_df(dep_var)
          .databunch(bs=128))
data_cls_mx.add_test(test_df)

print(f'CATS:\n{data_cls_mx.train_ds.cat_names}')
print(f'CONTS:\n{data_cls_mx.train_ds.cont_names}')
print(f'TEXT COLS:\n{data_cls_mx.train_ds.text_cols}')
print(f'PROCS:\n{data_cls_mx.train_ds.procs}')
len(data_cls_mx.train_ds.x.vocab.itos), len(data_cls_mx.valid_ds.x.vocab.itos)

data_cls_mx.show_batch()

from fastai.callbacks import hook_output
from fastai.text.learner import RNNTrainer, masked_concat_pool

tab_layers = [200, 100]
tabular_args = {
    'metrics': accuracy, 
    'ps': [.05, .03], 
    'emb_drop': 0.3, 
    #'y_range': [1, 5.1]
}

bptt = 70
emb_size = 400
n_hidden = 1150
n_layers = 3

max_seq = 70*20
drop_mult = 0.5

text_args = {
    'bptt': bptt, 
    'drop_mult': drop_mult,
    'max_len': max_seq
}
class TabularTextNN(nn.Module):
    def __init__(self, data, tab_layers, tab_args={}, text_args={}):
        super().__init__()
        
        tab_learner = tabular_learner(data, tab_layers, **tab_args)
        tab_learner.model.layers = tab_learner.model.layers[:-1]
        self.tabular_model = tab_learner.model

        text_class_learner = text_classifier_learner(data, AWD_LSTM, **text_args)
        text_class_learner.load_encoder(f'{lang}fine_tuned_enc')
        self.text_enc_model = list(text_class_learner.model.children())[0]
        
        self.bn_concat = nn.BatchNorm1d(400*3+100)
        
        self.lin = nn.Linear(400*3+100, 50)
        self.final_lin = nn.Linear(50, data.c)
        
    def forward(self, x_cat:Tensor, x_cont:Tensor, text_input:Tuple[Tensor,Tensor]) -> Tensor:
        # grab the tabular results minus the last layer
        tab_res = self.tabular_model.forward(x_cat, x_cont)

        # grab the results of the encoder
        raw_outputs, outputs, mask = self.text_enc_model(text_input)
        text_final_rnn_output = masked_concat_pool(outputs, mask) #outputs[-1][:,-1] # => (bs, 400*3)
  
        # do the concat pooling trick with the tabular and text encoder results
        x = torch.cat([tab_res, text_final_rnn_output], dim=1)
        x = self.bn_concat(x)
        
        # run the final results though a couple of linears
        x = self.final_lin(F.relu(self.lin(x)))
        
        # return the goods
        return x, raw_outputs, outputs
    
    def reset(self):
        self.text_enc_model.reset()

model = TabularTextNN(data_cls_mx, tab_layers, tabular_args, text_args)

learn = None; gc.collect()
learn = Learner(data_cls_mx, model, metrics=[accuracy])
learn.callbacks.append(RNNTrainer(learn, bptt))
learn.lr_find()
learn.recorder.plot()

learn.fit_one_cycle(1, 1e-2, wd=1e-2, pct_start=0.2, moms=(0.8, 0.7))
# learn.unfreeze()
learn.fit_one_cycle(5, 1e-2, wd=1e-2, pct_start=0.2, moms=(0.8, 0.7))<jupyter_output><empty_output><jupyter_text>Mixed tab+text learner<jupyter_code>def get_val_idxs(train,n_splits=20):
    np.random.seed(42)
    cv = KFold(n_splits=n_splits, shuffle=True, random_state=42)
    train_idxs, valid_idxs = next(cv.split(train))
    return train_idxs,valid_idxs

cat_cols = [
    'keyword', 
    'location', 
]

# cont_cols = [
#     'c', 
#     'funny',
#     'useful',
#     'user_average_stars',
#     'business_stars'
# ]
cont_cols = None
txt_cols = ['text']

dep_var = ['target']

procs = [FillMissing, Categorify]

train_idxs,val_idxs = get_val_idxs(train_df,n_splits=20)
# train_idxs,val_idxs = get_val_idxs(train_df,n_splits=5)
train_idxs,val_idxs
train_idxs.shape,val_idxs.shape
text_db = (TextList.from_df(train_df, path, vocab=data_lm.vocab, cols='text')
    .split_by_idx(val_idxs)
    .label_from_df(cols='target')
    .add_test(test_df)
    .databunch(bs=bs, num_workers=1))

tab_db = (TabularList.from_df(train_df, cat_cols, cont_cols, procs=procs, path=path)
                            .split_by_idx(val_idxs)
                            .label_from_df(cols='target')
                            .add_test(TabularList.from_df(test_df, cat_cols, cont_cols, txt_cols,path=path))
                            .databunch(bs=bs))

train_ds = ConcatDataset(tab_db.train_ds.x, text_db.train_ds.x, tab_db.train_ds.y)
valid_ds = ConcatDataset(tab_db.valid_ds.x, text_db.valid_ds.x, tab_db.valid_ds.y)
    
train_sampler = SortishSampler(text_db.train_ds.x, key=lambda t: len(text_db.train_ds[t][0].data), bs=bs//2)
valid_sampler = SortSampler(text_db.valid_ds.x, key=lambda t: len(text_db.valid_ds[t][0].data))

#     train_dl = DataLoader(train_ds, bs//2, sampler=train_sampler)
train_dl = DataLoader(train_ds, bs//2, sampler=train_sampler)
valid_dl = DataLoader(valid_ds, bs, sampler=valid_sampler)
data = DataBunch(train_dl, valid_dl, device=defaults.device, collate_fn=tabtext_collate, path=path)
doc(SortishSampler)
tab_params={
    'layers':[500],
    'emb_drop': 0.3,
    'y_range': [0,6],
    'use_bn': True,
    'ps': [0.4]
    }

text_params={
    #     'lin_ftrs':[1000],
         'ps': [0.6],
        'bptt':70,
        'max_len':20*70,
        'drop_mult': 0.5, 
    #     'use_bn': True,    
    }
# tab_learner = tabular_learner(data,metrics=[root_mean_squared_error],**params)
tab_learner = tabular_learner(tab_db,metrics=[accuracy,f1],**tab_params)
text_learner = text_classifier_learner(text_db,AWD_LSTM,metrics=[accuracy,f1],**text_params)
text_learner = text_learner.load_encoder('enfine_tuned_enc')
from tabtext import *
lin_layers=[500]
#ps=[0.7]

# 50 is the default lin_ftrs in AWD_LSTM
lin_layers[-1]+= 50 if 'lin_ftrs' not in text_params else text_params['lin_ftrs']

# be careful here. If no lin_ftrs is specified, the default lin_ftrs is 50

learner = get_tabtext_learner(data,tab_learner,text_learner,lin_layers ,ps)
learner.freeze()
learner.lr_find()
learner.recorder.plot(skip_end=1)
learner.freeze()
lr=2e-2
lr *= bs/48
learner.fit_one_cycle(5, lr, moms=(0.8, 0.7))
# learner.freeze_to(-2)
learner.unfreeze()

learner.fit_one_cycle(5, slice(1e-03/(2.6**4),1e-03), moms=(0.8, 0.7))
text_learner.model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15323691460055097, 'fraction_numerical': 0.03185261707988981, 'mean_word_length': 4.975823045267489, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '133931', 'license': 'Apache-2.0', 'n_tokens_mistral': 5165, 'n_tokens_neox': 4995, 'n_words': 660, 'path': 'kernel635a66b01d.ipynb', 'repo_name': 'shanmugamm212/Kernels', 'size': '11616'}