<jupyter_start><jupyter_text>CS155 Project 3 - Shakespearean Sonnets: RNNs**Author:** <NAME>**Description:** this notebook trains RNNs to write parody Shakespearean sonnets by training on a) all 154 Shakespearean sonnets in Section 2; b) both Shakespear's 154 poems and Edmund Spenser's Amoretti in Section 3.***And I name my Shakespearean sonnet-writing RNN to be "William-wanna-shake-pear".<jupyter_code>import re
import pickle
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams.update({'font.size': 12})

from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, LSTM, Lambda, Dropout
from keras.utils.vis_utils import plot_model
from keras.models import load_model
from keras.preprocessing.sequence import pad_sequences<jupyter_output><empty_output><jupyter_text>1. Load Pre-processed Data<jupyter_code># basic char-based datasets (40-char sequences) pre-processed from shakespeare.txt
basic_char_seqs_vec = pickle.load(open("processed_data/basic_char_seqs_vec.p", "rb"))
basic_char2vec = pickle.load(open("processed_data/basic_char2vec.p", "rb"))
basic_vec2char = pickle.load(open("processed_data/basic_vec2char.p", "rb"))
# advanced char-based datasets (40-char sequences) pre-processed from shakespeare.txt and spenser.txt
adv_char_seqs_vec = pickle.load(open("processed_data/adv_char_seqs_vec.p", "rb"))
adv_char2vec = pickle.load(open("processed_data/adv_char2vec.p", "rb"))
adv_vec2char = pickle.load(open("processed_data/adv_vec2char.p", "rb"))<jupyter_output><empty_output><jupyter_text>2. Poetry Generation - Recurrent Neural Network (Naive) Generate training data (X, Y) with the right shapes as inputs to the model.<jupyter_code>def gen_train_pair(char_seqs_vec, char2vec):
    char_seqs_vec = np.array(char_seqs_vec)
    x_seq, y_seq = char_seqs_vec[:, :-1], char_seqs_vec[:,-1]

    X = np.array([to_categorical(x, num_classes=len(char2vec)) for x in x_seq])
    Y = to_categorical(y_seq, num_classes=len(char2vec))
    return X, Y
basic_X, basic_Y = gen_train_pair(basic_char_seqs_vec, basic_char2vec)<jupyter_output><empty_output><jupyter_text>Define a simple model, consisting of 1 `LSTM` layer of 200 units and 1 `Dense` layer with `softmax` activation.<jupyter_code>basic_hidden_size = 200
basic_model = Sequential()
basic_model.add(LSTM(basic_hidden_size, input_shape=(basic_X.shape[1], basic_X.shape[2])))
basic_model.add(Dense(len(basic_char2vec), activation='softmax'))
basic_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

plot_model(basic_model, to_file='figures/basic_rnn_model.png', show_shapes=True)
basic_model.summary()<jupyter_output>Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm_1 (LSTM)                (None, 200)               184000    
_________________________________________________________________
dense_1 (Dense)              (None, 29)                5829      
=================================================================
Total params: 189,829
Trainable params: 189,829
Non-trainable params: 0
_________________________________________________________________<jupyter_text>Train a model for 60 epochs or load the trained model.<jupyter_code>train_model = False

# model training
if train_model:
    basic_model.fit(basic_X, basic_Y, batch_size=128, epochs=60)
    basic_model.save('processed_data/basic_char_rnn.h5')

# model loading if already trained
else:
    basic_model = load_model('processed_data/basic_char_rnn.h5')<jupyter_output><empty_output><jupyter_text>Add a `Lambda` layer to include a temperature param into the trained model for predictions.<jupyter_code>def get_model_with_temp(model, hidden_size, X, char2vec, temperatures, mode):
    # obtain model's trained weights
    model_weights = [layer.get_weights() for layer in model.layers]

    # construct models with different temperatures
    temp_models = []
    for temp in temperatures:
        if mode == 'basic':
            # add the Lambda(temperature) layer between LSTM and Dense
            m = Sequential([
                LSTM(hidden_size, input_shape=(X.shape[1], X.shape[2])),
                Lambda(lambda x: x / temp),
                Dense(len(char2vec), activation='softmax')
            ])
            m.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

            # assigned trained weights to the new temperature model
            m.layers[0].set_weights(model_weights[0])
            m.layers[2].set_weights(model_weights[1])

            temp_models.append(m)

        elif mode == 'adv':
            m = Sequential([
                LSTM(hidden_size, input_shape=(X.shape[1], X.shape[2]), return_sequences=True),
                LSTM(hidden_size, return_sequences=True),
                LSTM(hidden_size),
                Lambda(lambda x: x / temp),
                Dense(len(char2vec), activation='softmax')
            ])
            m.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
            
            # assigned trained weights to the new temperature model
            for i in range(3):
                m.layers[i].set_weights(model_weights[i*2])
            m.layers[-1].set_weights(model_weights[-1])
            
            temp_models.append(m)
            
        else:
            raise ValueError('Input should be either "basic" or "adv"')

    return temp_models
temperatures = [1.5, 0.75, 0.25]
basic_temp_models = get_model_with_temp(basic_model, basic_hidden_size, basic_X, 
                                        basic_char2vec, temperatures, 'basic')<jupyter_output><empty_output><jupyter_text>Generate a Shakespearean sonnet with the seed "*shall i compare thee to a summer's day*".<jupyter_code>def gen_poem(m, char2vec, X, seed, seq_len, n_line=14):
    
    poem = [seed.capitalize()]
    
    # generate the rest of the poem starting with seed
    for i in range(n_line-1):
        
        out_line = ''
        for _ in range(seq_len[i]):
            vecs = [char2vec[c] for c in seed]
            vecs = pad_sequences([vecs], maxlen=X.shape[1])
            vecs = to_categorical(vecs, num_classes=len(char2vec))
            
            # predict next vec
            y = m.predict_classes(vecs)
            
            # map vec to char
            for c, j in char2vec.items():
                if j == y[0]:
                    seed += c
                    out_line += c
                    break

        # remove the last word to get rid of incomplete words
        seed = ' '.join(seed.split(' ')[:-1])
        out_line = ' '.join(out_line.split(' ')[:-1])
        
        # add to poem
        poem.append(out_line.lstrip().capitalize())
    
    # print poem
    for j, line in enumerate(poem):
        words = line.split()
        
        # change 'i' into 'I' and 'o' into 'O'
        for i, w in enumerate(words):
            if w == 'i' or w == 'o':
                words[i] = w.upper()
        
        # print the poem
        if j % 4 != 0:
            print('  ' + ' '.join(words))
        else:
            print(' '.join(words))
        
## choose a sequence length with some randomness
# here the median sequence length is set to be 45 = 40 + 5, where
# the extra 5 is a buffer to get rid of the last word (chars after the last ' ')
med_seq_len = 40 + 5
n_line = 14
seq_len = np.random.normal(med_seq_len, 5, size=n_line)
seq_len = [int(s) for s in seq_len]

for i, temp in enumerate(temperatures):
    print('For a temperature of {}, one generated poem is:'.format(temp))
    print('================================================')
    gen_poem(basic_temp_models[i], basic_char2vec, basic_X,
             "shall i compare thee to a summer's day", seq_len, n_line=n_line)
    print()<jupyter_output>For a temperature of 1.5, one generated poem is:
================================================
Shall I compare thee to a summer's day
  Of you are to outin and for my self
  Rosel and all my friend and your true
  Love alane exes to the very was of hight
Do I not for my self I spend all plack
  Should do a lear that in the wrickle glacter
  Tell come how I am for while in their
  Putcons injures home and beauty looking
When in your sweet self to be so thus did
  The sime the deages ngrest and that beauty that I do
  Cold concest of the tears that I
  In other place or glasoul his
With the trouning that the thought of
  Hearts can beauty to the wirned some with

For a temperature of 0.75, one generated poem is:
================================================
Shall I compare thee to a summer's day
  Of you are to outin and for my self
  Rosel and all my friend and your true
  Love alane exes to the very was of hight
Do I not for my self I spend all plack
  Should do a lear that in th[...]<jupyter_text>3. Poetry Generation - RNN (Advanced) Train or load the advanced RNN model with 3 LSTM layers of 600 units.<jupyter_code>train_model = False

adv_hidden_size = 600
adv_X, adv_Y = gen_train_pair(adv_char_seqs_vec, adv_char2vec)

# model training
if train_model:
    adv_model = Sequential([
        LSTM(adv_hidden_size, input_shape=(adv_X.shape[1], adv_X.shape[2]), return_sequences=True),
        Dropout(0.2),
        LSTM(adv_hidden_size, return_sequences=True),
        Dropout(0.2),
        LSTM(adv_hidden_size),
        Dropout(0.2),
        Dense(len(adv_char2vec), activation='softmax')
    ])
    adv_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

    plot_model(adv_model, to_file='processed_data/adv_rnn_model.png', show_shapes=True)
    adv_model.summary()
    
    adv_model.fit(adv_X, adv_Y, batch_size=128, epochs=40)
    adv_model.save('processed_data/adv_char_rnn.h5')

# model loading if already trained
else:
    adv_model = load_model('processed_data/adv_char_rnn.h5')
    adv_model.summary()
adv_temp_models = get_model_with_temp(adv_model, adv_hidden_size, adv_X, 
                                      adv_char2vec, temperatures, 'adv')<jupyter_output><empty_output><jupyter_text>Generate 3 poems with a temperature of 0.75 using different seeds with this advanced RNN model.<jupyter_code>def gen_poem_adv(m, char2vec, X, seed, max_char=800):
    # generate max_char number of chars from seed
    for _ in range(max_char):
        vecs = [char2vec[c] for c in seed]
        vecs = pad_sequences([vecs], maxlen=X.shape[1])
        vecs = to_categorical(vecs, num_classes=len(char2vec))

        # predict next vec
        y = m.predict_classes(vecs)

        # map vec to char
        for c, j in char2vec.items():
            if j == y[0]:
                seed += c
                break
        
    # generate poem of 14 lines
    poem = []
    str_split = re.split('(?<=[!?,.;:]) +', seed)
    for i, s in enumerate(str_split):
        if i == 0:
            poem.append((s + '\n').capitalize())
        else:
            if len(poem[-1]) > 20:
                if len(s) > 20:
                    poem.append((s + '\n').capitalize())
                else:
                    poem.append((s + ' ').capitalize())
            else:
                poem[-1] += (s + '\n')
    
    # formatting
    for i in range(14):
        if i % 4 != 0:
            poem[i] = '  ' + poem[i]
        if i == 13:    
            poem[i] = '  ' + poem[i][:-2] + '.'
    
    print(''.join(poem[:14]))
gen_poem_adv(adv_temp_models[1], adv_char2vec, adv_X,
             "great was the glory then gained in the fight.")
gen_poem_adv(adv_temp_models[1], adv_char2vec, adv_X,
             "then the powerful king put to the test,")
gen_poem_adv(adv_temp_models[1], adv_char2vec, adv_X,
             "my robe is noiseless when i roam the earth,")<jupyter_output>My robe is noiseless when i roam the earth,
  Which her fair child expire,
  Shall to a baser made,
  And soon to temper that my wit cannot endite.
When suddenly with thine eye but with thy time decays?
  O fearful meditation,
  Where alack, shall time's best jewel will be took,
  And they that skill not of the world's fresh care,
And me for me than shortune your self still,
  And therefore to be seen so thy great growing age,
  A dearer birth than thine eyes seem so.
  If it be not, the which three times thrice haply hath me,
Suffine in them shall still will play the tyrant,
    The which beholding me with melancholy.
{'fraction_non_alphanumeric': 0.11961989938513136, 'fraction_numerical': 0.01621017328116266, 'mean_word_length': 3.0821382007822686, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '228763', 'license': 'MIT', 'n_tokens_mistral': 4074, 'n_tokens_neox': 3851, 'n_words': 1130, 'path': '3-RNN-William-Wanna-Shake-Pear.ipynb', 'repo_name': 'litingxiao/william-wanna-shake-pear', 'size': '12528'}