<jupyter_start><jupyter_text>Language model<jupyter_code>wd=1e-7
bptt=70
bs=100
#opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
opt_fn = partial(optim.SGD, momentum=0.9)
weight_factor = 0.1
bs=100
trn_dl = LanguageModelLoader(np.concatenate(trn_lm), bs, bptt)
val_dl = LanguageModelLoader(np.concatenate(val_lm), bs, bptt)
md = LanguageModelData(PATH_MODELS, 1, vs, trn_dl, val_dl, bs=bs, bptt=bptt)
drops = np.array([0.25, 0.1, 0.2, 0.02, 0.15])*weight_factor
learner= md.get_model(opt_fn, em_sz, nh, nl, 
    dropouti=drops[0], dropout=drops[1], wdrop=drops[2], dropoute=drops[3], dropouth=drops[4])

learner.metrics = [accuracy]
learner.freeze_to(-1)
learner.model.load_state_dict(wgts)
# AWD-LSTM network
learner.summary
lr=1e-3
lrs = lr
%time 

learner.fit(lrs/2, 1, wds=wd, use_clr=(32,2), cycle_len=1)
learner.save('lm_last_ft')
learner.load('lm_last_ft')
learner.unfreeze()
%time

learner.lr_find2(num_it=1000)
learner.sched.plot()
# Set learning rate
lr = 0.2
%time

learner.fit(lr, 1, wds=wd, cycle_len=50, use_clr=(10,33,0.95,0.85), best_save_name='best_lm_indonesia_1cycle_bppt_panl')
%time

learner.fit(lr, 1, wds=wd, cycle_len=50, use_clr=(10,33,0.95,0.85), best_save_name='best_lm_indonesia_1cycle_bppt_panl')
learner.sched.plot_loss()
#learner.lr_find(start_lr=lrs/10, end_lr=lrs*10, linear=True)
#learner.sched.plot()
#learner.fit(lrs, 1, wds=wd, use_clr=(20,10), cycle_len=15)
learner.save('lm1')
learner.save_encoder('lm1_enc')
learner.sched.plot_loss()<jupyter_output><empty_output><jupyter_text>Classifier tokens<jupyter_code>chunksize = 10000
df_trn = pd.read_csv(f"{PATH_DATASET}/bppt_panl_train.csv", header=None, chunksize=chunksize)
df_val = pd.read_csv(f"{PATH_DATASET}/bppt_panl_test.csv", header=None, chunksize=chunksize)
tok_trn, trn_labels = get_all(df_trn, 1)
tok_val, val_labels = get_all(df_val, 1)
(PATH_TMP).mkdir(exist_ok=True)

np.save(PATH_TMP/'bppt_panl_tok_trn.npy', tok_trn)
np.save(PATH_TMP/'bppt_panl_tok_val.npy', tok_val)

np.save(PATH_TMP/'bppt_panl_trn_labels.npy', trn_labels)
np.save(PATH_TMP/'bppt_panl_val_labels.npy', val_labels)
tok_trn = np.load(PATH_TMP/'bppt_panl_tok_trn.npy')
tok_val = np.load(PATH_TMP/'bppt_panl_tok_val.npy')
itos = pickle.load((PATH_MODELS/'bppt_panl_itos.pkl').open('rb'))
stoi = collections.defaultdict(lambda:0, {v:k for k,v in enumerate(itos)})
len(itos)
trn_clas = np.array([[stoi[o] for o in p] for p in tok_trn])
val_clas = np.array([[stoi[o] for o in p] for p in tok_val])
np.save(PATH_TMP/'bppt_panl_trn_ids.npy', trn_clas)
np.save(PATH_TMP/'bppt_panl_val_ids.npy', val_clas)<jupyter_output><empty_output><jupyter_text>Classifier<jupyter_code>trn_clas = np.load(PATH_TMP/'bppt_panl_trn_ids.npy')
val_clas = np.load(PATH_TMP/'bppt_panl_val_ids.npy')
trn_labels = np.squeeze(np.load(PATH_TMP/'bppt_panl_trn_labels.npy'))
val_labels = np.squeeze(np.load(PATH_TMP/'bppt_panl_val_labels.npy'))
bptt,em_sz,nh,nl = 70,400,1150,3
vs = len(itos)
opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
bs = 48
weight_factor = 0.8
min_lbl = trn_labels.min()
trn_labels -= min_lbl
val_labels -= min_lbl
c=int(trn_labels.max())+1
trn_ds = TextDataset(trn_clas, trn_labels)
val_ds = TextDataset(val_clas, val_labels)
trn_samp = SortishSampler(trn_clas, key=lambda x: len(trn_clas[x]), bs=bs//2)
val_samp = SortSampler(val_clas, key=lambda x: len(val_clas[x]))
trn_dl = DataLoader(trn_ds, bs//2, transpose=True, num_workers=1, pad_idx=1, sampler=trn_samp)
val_dl = DataLoader(val_ds, bs, transpose=True, num_workers=1, pad_idx=1, sampler=val_samp)
md = ModelData(PATH_MODELS, trn_dl, val_dl)
# part 1
dps = np.array([0.4, 0.5, 0.05, 0.3, 0.1])
dps = np.array([0.4,0.5,0.05,0.3,0.4])*weight_factor
m = get_rnn_classifier(bptt, 20*70, c, vs, emb_sz=em_sz, n_hid=nh, n_layers=nl, pad_token=1,
          layers=[em_sz*3, 50, c], drops=[dps[4], 0.1],
          dropouti=dps[0], wdrop=dps[1], dropoute=dps[2], dropouth=dps[3])
opt_fn = partial(optim.Adam, betas=(0.7, 0.99))
learn = RNN_Learner(md, TextModel(to_gpu(m)), opt_fn=opt_fn)
learn.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)
learn.clip=.25
learn.metrics = [accuracy]
lr=3e-3
lrm = 2.6
lrs = np.array([lr/(lrm**4), lr/(lrm**3), lr/(lrm**2), lr/lrm, lr])
lrs=np.array([1e-4,1e-4,1e-4,1e-3,1e-2])
wd = 1e-7
wd = 0
learn.load_encoder('lm1_enc')
learn.freeze_to(-1)
learn.lr_find(lrs/1000)
learn.sched.plot()
learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8,3))
learn.save('clas_0')
learn.load('clas_0')
learn.freeze_to(-2)
learn.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8,3))
learn.save('clas_1')
learn.load('clas_1')
learn.unfreeze()
%time

learn.fit(lrs, 1, wds=wd, cycle_len=20, use_clr=(32,10))
learn.sched.plot_loss()
learn.save('clas_2')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1756440281030445, 'fraction_numerical': 0.04769001490312966, 'mean_word_length': 7.285714285714286, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '320799', 'license': 'MIT', 'n_tokens_mistral': 2349, 'n_tokens_neox': 2285, 'n_words': 311, 'path': 'ULMFiT/ulmfit_classification.ipynb', 'repo_name': 'cahya-wirawan/language-modelling', 'size': '4697'}