<jupyter_start><jupyter_text>Data<jupyter_code>wiki = 'https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip'
path = untar_data(wiki)
Path.BASE_PATH = path
# !mv {path}/wiki.valid.tokens {path}/valid.txt
# !mv {path}/wiki.test.tokens {path}/test.txt
# !mv {path}/wiki.train.tokens {path}/train.txt
path.ls()
def istitle(line):
    return len(re.findall(r'^ = [^=]* = $', line)) != 0
def read_wiki(filename):
    articles = []
    with open(filename, encoding='utf8') as f:
        lines = f.readlines()
    current_article = ''
    for i,line in enumerate(lines):
        current_article += line
        if i < len(lines) - 2 and lines[i+1] == ' \n' and istitle(lines[i+2]):
            current_article = current_article.replace('<unk>', UNK)
            articles.append(current_article)
            current_article = ''
    current_article = current_article.replace('<unk>', UNK)
    articles.append(current_article)
    return articles
articles = read_wiki(path/'train.txt')
len(articles)
train = TextList(read_wiki(path/'train.txt'), path)
valid = TextList(read_wiki(path/'valid.txt'))
len(train), len(valid)
sd = SplitData(train, valid)
proc_tok, proc_num = TokenizeProcessor(max_workers=1), NumericalizeProcessor()
ll = label_by_func(sd, lambda x: 0, proc_x=[proc_tok, proc_num])
pickle.dump(ll, open(path/'ld.pkl', 'wb'))
ll = pickle.load(open(path/'ld.pkl', 'rb'))
bs,bptt = 128,70
data = lm_databunchify(ll, bs, bptt)
vocab = ll.train.proc_x[-1].vocab
len(vocab)
ll.train.proc_x<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>dps = np.array([0.1, 0.15, 0.25, 0.02, 0.2]) * 0.2
tok_pad = vocab.index(PAD)
emb_sz, nh, nl = 300, 300, 2
model = get_language_model(len(vocab), emb_sz, nh, nl, tok_pad, *dps)
cbs = [partial(AvgStatsCallback, accuracy_flat),
      CudaCallback, Recorder, ProgressCallback,
      partial(GradientClipping, 0.1),
      partial(RNNTrainer, α=2., β=1.)]
learn = Learner(model, data, cross_entropy_flat, lr=5e-3, cb_funcs=cbs, opt_func=adam_opt())
lr = 5e-3
sched_lr = combine_scheds([0.3, 0.7], cos_1_cycle_anneal(lr/10., lr, lr/1e5))
sched_mom = combine_scheds([0.3, 0.7], cos_1_cycle_anneal(0.8, 0.7, 0.8))
cbsched = [ParamScheduler('lr', sched_lr), ParamScheduler('mom', sched_mom)]
learn.fit(2, cbs=cbsched)
torch.save(learn.model.state_dict(), path/'pretrained.pth')
pickle.dump(vocab, open(path/'vocab.pkl', 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17376913529168392, 'fraction_numerical': 0.028547786512205214, 'mean_word_length': 4.941031941031941, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '455932', 'license': 'Apache-2.0', 'n_tokens_mistral': 1002, 'n_tokens_neox': 954, 'n_words': 164, 'path': 'nbs/dl2/selfmade/12b_lm_pretrain.ipynb', 'repo_name': 'Hustens0hn/fastai-course-v3', 'size': '2417'}