<jupyter_start><jupyter_text>Generate sample text segments for use of language model and for predictionNote: intended to run on 8GB+ CUDA-capable machine<jupyter_code>%matplotlib inline
import pymongo
import random
import numpy as np

random.seed(42)
MONGODB_PORT = 25541

# return this percent of the snippets
snippetSamplePct = 0.01
assert(snippetSamplePct < 1.)
output_filename = 'snippets_sample.1pct.tsv' # even though there's only one field, call it tab-seperated, because commas are in the data.

# create a sample of p% of the data, excluding training and holdout sets
with pymongo.MongoClient('localhost', MONGODB_PORT) as mclient:
    mdb = mclient.snippetdb
    msnippets = mdb.snippets
    #msnippets.create_index([('holdout', pymongo.ASCENDING)])
    #msnippets.create_index([('train_set', pymongo.ASCENDING)])
    
    #nSnippets = msnippets.count_documents({'holdout': {'$exists': False}, 'train_set': {'$exists': False}})
    #bernoulli_draw = np.random.binomial(size=nSnippets, n=1, p = 0.1)
    
    doc_cursor = msnippets.find({'holdout': {'$exists': False}, 'train_set': {'$exists': False}})
    
    with open(output_filename, 'w') as snippet_file:
        for i, doc in enumerate(doc_cursor):
            if i%10000 == 0: 
                print('processing record ' + str(i))
            if np.random.binomial(1, snippetSamplePct):
                # bernoulli draw comes up heads, save this one
                snippet = " ".join([doc['snippet_part1'], doc['snippet_part2'], doc['snippet_part3']])
                snippet_file.write(snippet + '\n')<jupyter_output><empty_output><jupyter_text>Train a language model based upon this snippet sample<jupyter_code>from fastai.text import *
bs = 64 # size of minibatch
# written with fastai v1.0.42<jupyter_output><empty_output><jupyter_text>TODO: increase vocab size? fine tune tokenization?    current method uses 6510/8110 MB video ram    Note: this code runs on demon<jupyter_code># Yay, my new way works!
# (this is using an addition to fastai that will be incorporated in 1.0.44)
data_lm_test = TextLMDataBunch.from_csv('snippet_samples', sample_filename, header=None, delimiter='\t', col_names=['text'], valid_pct=0.005, min_freq=4, text_cols=['text'])
# this is the way that works in v1.0.42 of fastai
valid_pct = 0.005
path = 'snippet_samples'
df = pd.read_csv(path + '/' + sample_filename, header = None, delimiter='\t', names=['text'])
#df = pd.read_csv(Path(path)/csv_name, header=header)
df = df.iloc[np.random.permutation(len(df))]
cut = int(valid_pct * len(df)) + 1
train_df, valid_df = df[cut:], df[:cut]

data_lm = TextLMDataBunch.from_df(path, train_df, valid_df, text_cols='text', min_freq=4)
data_lm.save('data_lm')
print(data_lm.vocab.itos[:15])
print(len(data_lm.vocab.itos))
data_lm.show_batch()
# Transfer learning from wikipedia-trained LM. drop_mult reduced from 1 to avoid underfitting.
learn = language_model_learner(data_lm, pretrained_model=URLs.WT103_1, drop_mult=0.5)
learn.lr_find()
learn.recorder.plot(skip_end=10)
learn.fit_one_cycle(2, max_lr=3e-2, moms=(0.8,0.7)) # wd=0.01
learn.save('lm_fit_head2')
learn.unfreeze()
learn.lr_find()
learn.recorder.plot(skip_end=30)
learn.fit_one_cycle(2, max_lr=slice(8e-6, 2e-2), moms=(0.85,0.75))
learn.save('lm_fine_tune2_drop_mult0.5_2epoch')
learn.fit_one_cycle(4, max_lr=slice(8e-6, 2e-2), moms=(0.85,0.75))
learn.save('lm_fine_tune2_drop_mult0.5_6epoch')
TEXT = "> I just didn't"
N_WORDS = 40
N_SENTENCES = 2
print("\n".join(learn.predict(TEXT, N_WORDS, temperature=0.75) for _ in range(N_SENTENCES)))
learn.fit_one_cycle(6, max_lr=slice(8e-6, 2e-2), moms=(0.85,0.75))
learn.save('lm_fine_tune2_drop_mult0.5_12epoch')
learn = language_model_learner(data_lm, pretrained_model=URLs.WT103_1)
learn.lr_find()
learn.recorder.plot(skip_end=15)
learn.fit_one_cycle(2, max_lr=2e-2, moms=(0.8,0.7)) # wd=0.01
learn.save('lm_fit_head3')
learn.unfreeze()
learn.lr_find()
learn.recorder.plot(skip_end=25)
learn.fit_one_cycle(5, max_lr=slice(8e-4, 2e-2), moms=(0.85,0.75))
learn.save('lm_fine_tune3_5epoch')
learn.fit_one_cycle(1, max_lr=slice(8e-4, 2e-2), moms=(0.8,0.7))
# save encoder from the learner (the part that learns the language)
learn = language_model_learner(data_lm, pretrained_model=URLs.WT103_1) #drop_multi=0.3
learn.load('lm_fine_tune3_5epoch')
learn.save_encoder('lm_fine_tuned3_enc')<jupyter_output><empty_output><jupyter_text>Now use the language model's encoder to improve classification.  Train a new classifier from our labeled data<jupyter_code>data_lm = TextLMDataBunch.load('snippet_samples', 'data_lm')
# these approaches won't work...

# data_clas = (TextList.from_folder('labeled_data/label_category', vocab=data_lm.vocab)
#              #grab all the text files in pat
#              .split_by_fname_file('label_category_valid.tsv')
#              #split by train and valid folder (that only keeps 'train' and 'test' so no need to filter)
#              #.label_from_folder(classes=['neg', 'pos'])
#              .label_from_df(cols=1)
#              #label them all with their folders
#              .databunch(bs=bs))

# data_clas = TextLMDataBunch.from_csv('labeled_data/label_category', 'label_category_train.tsv', 'label_category_valid.tsv')
# load our labeled data into a TextClasDataBunch
path = Path('labeled_data')
fold = '0'
path /= 'fold_' + fold
groupname = 'label_category'
train_df = pd.read_csv(path/groupname/(groupname+'_train-truecase.tsv'), header=None, delimiter='\t', names=['label','text'])
valid_df = pd.read_csv(path/groupname/(groupname+'_valid-truecase.tsv'), header=None, delimiter='\t', names=['label','text'])

data_clas = TextClasDataBunch.from_df(path, train_df, valid_df, vocab=data_lm.vocab, text_cols='text', label_cols='label', bs=bs)
data_clas.save('fold_'+fold+'-'+groupname+'_clas')
train_df.head()
valid_df.head()
data_clas.show_batch()
learn = text_classifier_learner(data_clas, drop_mult=1.0)
learn.load_encoder('../../../snippet_samples/models/lm_fine_tuned3_enc')
learn.freeze()
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))
learn.save('first')
learn.freeze_to(-2)
learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.85,0.75))
learn.save('second')
learn.load('second');
learn.freeze_to(-3)
learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.85,0.75))
learn.save('third')
learn.load('third');
learn.unfreeze()
learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))
learn.save('fine_tuned')
learn.load('fine_tuned')
learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))
learn.save('fine_tuned2')
learn.load('fine_tuned2')
# from prior run, this was declining
learn.fit_one_cycle(2, slice(5e-4/(2.6**4),5e-4), moms=(0.8,0.7))
learn.save('fine_tuned3')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15465443244043214, 'fraction_numerical': 0.03936658280301909, 'mean_word_length': 5.479386385426654, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '282007', 'license': 'MIT', 'n_tokens_mistral': 2748, 'n_tokens_neox': 2620, 'n_words': 485, 'path': 'fastai/LearnLanguageModel.ipynb', 'repo_name': 'jtbr/tv-news-quality', 'size': '6757'}