<jupyter_start><jupyter_text>textgenrnn* https://github.com/minimaxir/textgenrnn* https://github.com/minimaxir/textgenrnn/blob/master/docs/textgenrnn-demo.ipynb<jupyter_code>import sys, os
import pandas as pd
data_folder = '/media/student/8d1913cf-1155-47a5-a7db-b9a51f445d8f/student/data'
os.path.exists(data_folder)
os.listdir(data_folder)
reviews = pd.read_csv(
    data_folder + '/restaurant_reviews.csv',
    nrows=10000,
    sep=',',
    index_col=0,
    usecols=['stars', 'text']
)
reviews.reset_index(inplace=True)
reviews.head()
reviews.head()
reviews[reviews['stars'].isin([4, 5])]['text'].values
reviews.shape
from textgenrnn import textgenrnn
textgen = textgenrnn()
dir(textgen)
stars = [1, 2, 3, 4, 5]
texts = {
    star: reviews[reviews.stars == star]['text'].values[:20]
    for star in stars
}
models_pretrained = {
    star: textgenrnn(name='model_pretrained_{}'.format(star))
    for star in stars
}
models_new = {
    star: textgenrnn(name='model_new_{}'.format(star))
    for star in stars
}
def train_models(word_level, star2model, star2texts, num_epochs=1, gen_epochs=1, max_length=10, **kwargs):
    for star, model in star2model.items():
        model.train_on_texts(
            star2texts[star],
            word_level=word_level,
            num_epochs=num_epochs,
            gen_epochs=gen_epochs,
            max_length=max_length,
            **kwargs)
def train_models_word_level(star2model, star2texts, num_epochs=10, gen_epochs=5, max_length=40):
    train_models(True,
                 star2model,
                 star2texts,
                 num_epochs,
                 gen_epochs,
                 max_length,
                 **kwargs)
def train_models_char_level(star2model, star2texts, num_epochs=80, gen_epochs=10, max_length=10, **kwargs):
    train_models(False,
                 star2model,
                 star2texts,
                 num_epochs,
                 gen_epochs,
                 max_length
                 **kwargs)
train_models_word_level(
    models_pretrained,
    texts,
    num_epochs=10,
    gen_epochs=5)
train_models_word_level(
    models_new,
    texts,
    num_epochs=10,
    gen_epochs=5)
t = models_pretrained[1].generate(2, return_as_list=True)
t
models_pretrained[1].save()
tt = textgenrnn(name='model_pretrained_1')


tt.reset()
texts = ['Never gonna give you up, never gonna let you down',
            'Never gonna run around and desert you',
            'Never gonna make you cry, never gonna say goodbye',
            'Never gonna tell a lie and hurt you']
tt.train_on_texts(texts, num_epochs=1, new_model=True)
dir(tt)
tt.config
textgen.generate(1, max_gen_length=100)<jupyter_output>I was something peopt this pot that could he wanted a reason and a brind of the service was always
{'fraction_non_alphanumeric': 0.10994575045207956, 'fraction_numerical': 0.027848101265822784, 'mean_word_length': 3.1469265367316344, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '164023', 'license': 'MIT', 'n_tokens_mistral': 1000, 'n_tokens_neox': 983, 'n_words': 195, 'path': 'notebooks/main.ipynb', 'repo_name': 'lingofunk/forumite-generate', 'size': '2765'}