<jupyter_start><jupyter_text>Thai Wiki Language Model for Text Generation This notebook details how you can use pretrained language model on [Thai Wikipedia Dump](https://dumps.wikimedia.org/thwiki/latest/thwiki-latest-pages-articles.xml.bz2) to generate texts.<jupyter_code># #uncomment if you are running from google colab
# !pip install sklearn_crfsuite
# !pip install https://github.com/PyThaiNLP/pythainlp/archive/dev.zip
# !pip install fastai==1.0.46
# !pip install emoji
import pandas as pd
import numpy as np
from ast import literal_eval
from tqdm import tqdm_notebook
from collections import Counter
import re

#viz
import matplotlib.pyplot as plt
import seaborn as sns

#fastai
import fastai
from fastai.text import *
from fastai.callbacks import CSVLogger

#pythainlp
from pythainlp.ulmfit import *
from pythainlp.tokenize import word_tokenize

#utils
def pretty_print(t, limit = 20):
    words = word_tokenize(t, engine='ulmfit')
    line = []
    nb_words = 0
    for w in words:
        if w == '\n': continue
        line.append(w)
        nb_words+=1
        if nb_words > limit:
            print(''.join(line))
            line = []
            nb_words = 0
#get dummy data
imdb = untar_data(URLs.IMDB_SAMPLE)
dummy_df = pd.read_csv(imdb/'texts.csv')

#get vocab
thwiki_itos = pickle.load(open(_THWIKI_LSTM['itos_fname'],'rb'))
thwiki_vocab = fastai.text.transform.Vocab(thwiki_itos)

#dummy databunch
tt = Tokenizer(tok_func = ThaiTokenizer, lang = 'th', pre_rules = pre_rules_th, post_rules=post_rules_th)
processor = [TokenizeProcessor(tokenizer=tt, chunksize=10000, mark_fields=False),
            NumericalizeProcessor(vocab=thwiki_vocab, max_vocab=60000, min_freq=3)]
data_lm = (TextList.from_df(dummy_df, imdb, cols=['text'], processor=processor)
    .random_split_by_pct(0.2)
    .label_for_lm()
    .databunch(bs=64))


data_lm.sanity_check()
#check vocab size
len(data_lm.vocab.itos)
config = dict(emb_sz=400, n_hid=1550, n_layers=4, pad_token=1, qrnn=False, tie_weights=True, out_bias=True,
             output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15)
trn_args = dict(drop_mult=0.9, clip=0.12, alpha=2, beta=1)

learn = language_model_learner(data_lm, AWD_LSTM, config=config, pretrained=False, **trn_args)

#load pretrained models
learn.load_pretrained(**_THWIKI_LSTM)
t = learn.predict('กาลครั้งหนึ่งนานมาแล้ว', 200, temperature=0.8, sep = '')
pretty_print(t)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11543184609087188, 'fraction_numerical': 0.024150634465820713, 'mean_word_length': 4.84688995215311, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '291457', 'license': 'MIT', 'n_tokens_mistral': 963, 'n_tokens_neox': 921, 'n_words': 198, 'path': 'thwiki_lm/thwiki_lm_example.ipynb', 'repo_name': 'cstorm125/thai2fit', 'size': '2443'}