<jupyter_start><jupyter_text>Generating text with LSTMs<jupyter_code># Run only the first time
import nltk
nltk.download('gutenberg')
import numpy as np
import random
import sys

from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.layers import LSTM
from keras.optimizers import RMSprop

from nltk.corpus import gutenberg
gutenberg.fileids()<jupyter_output><empty_output><jupyter_text>Go to [Project Gutenberg](https://www.gutenberg.org) if you want more<jupyter_code>text = ''
for txt in gutenberg.fileids():
    if 'shakespeare' in txt:
        text += gutenberg.raw(txt).lower()
chars = sorted(list(set(text)))

# dictionary from character to index
char_indices = dict((c, i) for i, c in enumerate(chars))

# distionary from index to character
indices_char = dict((i, c) for i, c in enumerate(chars))

print('corpus length: {} total chars: {}'.format(len(text), len(chars)))

print(text[:500])<jupyter_output><empty_output><jupyter_text>**Objective.** Predicting the character after having seen 40 characters**Trick.** Adding redundancy to the training collection<jupyter_code>maxlen = 40
step = 3
sentences = []
next_chars = []
# Notice: no tokenisation; no sentence splitting; no linebreak elimination
for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i: i + maxlen])
    next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))
# Producing the one-hot encoding
X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)

for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1
# Building the model

model = Sequential()
model.add(LSTM(128,
    input_shape=(maxlen, len(chars))))

model.add(Dense(len(chars)))
model.add(Activation('softmax'))

# https://keras.io/api/optimizers/rmsprop/; lr=learning rate (default: 0.001)
optimizer = RMSprop(lr=0.01)

# no more binary cross entropy; no dropout
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
model.summary()

# Training 
epochs = 6
batch_size = 128
model_structure = model.to_json()

with open("shakes_lstm_model.json", "w") as json_file:
    json_file.write(model_structure)

for i in range(5):
    model.fit(X, y,
        batch_size=batch_size,
        epochs=epochs
    )
    model.save_weights("shakes_lstm_weights_{}.h5".format(i+1))
    
# It should take about 25 epochs<jupyter_output><empty_output><jupyter_text>**Temperature**temperature > 1 : more diverse outcometemperature < 1 : more strict (try to "copy")<jupyter_code>def sample(preds, temperature=1.0):
    """Sampler to generate character sequences"""
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    
    # produces a number of random outcomes, 
    # given a probability distribution
    # n=1    number of experiments
    # preds  sequence of probabilities
    # size=1 
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)
start_index = random.randint(0, len(text) - maxlen - 1)
for diversity in [0.2, 0.5, 1.0, 1.2]:
    print()
    print('----- diversity:', diversity)
    # Getting a random starting text
    sentence = text[start_index: start_index + maxlen]
    generated = ''
    generated += sentence
    
    print('----- Generating with seed: "' + sentence + '"')
    sys.stdout.write(generated)
    for i in range(400):
        # one-hot representation
        x = np.zeros((1, maxlen, len(chars)))
        for t, char in enumerate(sentence):
            x[0, t, char_indices[char]] = 1.
        
        # Producing the prediction
        preds = model.predict(x, verbose=0)[0]
        next_index = sample(preds, diversity)
        
        # looking up the next character and adding it 
        next_char = indices_char[next_index]
        generated += next_char
        
        #updating the seed
        sentence = sentence[1:] + next_char
        
        sys.stdout.write(next_char)
        sys.stdout.flush()  # to display it right away
    print()
    
# lower values should look "more Shakesperean"
# Compiling the network
model.compile('rmsprop', 
              'binary_crossentropy', 
              metrics=['accuracy'])
print(model.summary())<jupyter_output><empty_output><jupyter_text>That's **70,200** parameters in the LSTM (against **17,550** for the RNN)Back to the slides<jupyter_code># Training the network

model.fit(x_train, y_train,
    batch_size=batch_size,
    epochs=epochs,
    validation_data=(x_test, y_test))
# Saving the network for future use
model_structure = model.to_json()
with open("lstm_model1.json", "w") as json_file:
    json_file.write(model_structure)
model.save_weights("lstm_weights1.h5")
# Predicting
sample_1 = """I hate that the dismal weather had me down for so long, when
will it break! Ugh, when does happiness return? The sun is blinding and
the puffy clouds are too thin. I can't wait for the weekend."""

vec_list = tokenize_and_vectorize([(1, sample_1)])
test_vec_list = pad_trunc(vec_list, maxlen)
test_vec = np.reshape(
    test_vec_list, 
    (len(test_vec_list), maxlen, embedding_dims))
model.predict_classes(test_vec)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12142723706332897, 'fraction_numerical': 0.014010835045768728, 'mean_word_length': 4.060491493383743, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '153891', 'license': 'MIT', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1769, 'n_words': 465, 'path': 'files/coli/week_10/18_dit_coli_lstm-gen.ipynb', 'repo_name': 'albarron/academic-kickstart', 'size': '5353'}