<jupyter_start><jupyter_text>RNN and LSTM AssignmentIt is said that [infinite monkeys typing for an infinite amount of time](https://en.wikipedia.org/wiki/Infinite_monkey_theorem) will eventually type, among other things, the complete works of Wiliam Shakespeare. Let's see if we can get there a bit faster, with the power of Recurrent Neural Networks and LSTM.This text file contains the complete works of Shakespeare: https://www.gutenberg.org/files/100/100-0.txtUse it as training data for an RNN - you can keep it simple and train character level, and that is suggested as an initial approach.Then, use that trained RNN to generate Shakespearean-ish text. Your goal - a function that can take, as an argument, the size of text (e.g. number of characters or lines) to generate, and returns generated text of that size.Note - Shakespeare wrote an awful lot. It's OK, especially initially, to sample/use smaller data and parameters, so you can have a tighter feedback loop when you're trying to get things running. Then, once you've got a proof of concept - start pushing it more! Stretch goals:- Refine the training and generation of text to be able to ask for different genres/styles of Shakespearean text (e.g. plays versus sonnets)- Train a classification model that takes text and returns which work of Shakespeare it is most likely to be from- Make it more performant! Many possible routes here - lean on Keras, optimize the code, and/or use more resources (AWS, etc.)- Revisit the news example from class, and improve it - use categories or tags to refine the model/generation, or train a news classifier- Run on bigger, better data<jupyter_code>import numpy as np


class RNN_NLP_Generator(object):
    """RNN with one hidden layer"""
    def __init__(self):
        pass

    def forward_prop(self, inputs, targets, h_prev):
        xs, hs, ys, ps = {}, {}, {}, {}
        hs[-1] = np.copy(h_prev)
        loss = 0

        for t in range(len(inputs)): # t is a "time step" and is used as a dict key
            xs[t] = np.zeros((self.num_chars,1))
            xs[t][inputs[t]] = 1
            hs[t] = np.tanh(np.dot(self.W_xh, xs[t]) + np.dot(self.W_hh, hs[t-1]) + self.b_h)
            ys[t] = np.dot(self.W_hy, hs[t]) + self.b_y # unnormalized log probabilities for next chars
            ps[t] = np.exp(ys[t]) / np.sum(np.exp(ys[t])) # probabilities for next chars. 

            # Softmax
            loss += -np.log(ps[t][targets[t],0]) # softmax (cross-entropy loss). Efficient and simple code

        return loss, ps, hs, xs

    def backward_prop(self, ps, inputs, hs, xs, targets):
        # make all zero matrices
        dWxh, dWhh, dWhy, dbh, dby, dhnext = \
            [np.zeros_like(_) for _ in [self.W_xh, self.W_hh, self.W_hy, 
                                        self.b_h,  self.b_y,  hs[0]]]
        # reversed
        for t in reversed(range(len(inputs))):
            dy = np.copy(ps[t]) # shape (num_chars,1).  "dy" means "dloss/dy"
            dy[targets[t]] -= 1 # backprop into y. After taking the soft max in the input vector, subtract 1 from the value of the element corresponding to the correct label.
            dWhy += np.dot(dy, hs[t].T)
            dby += dy 
            dh = np.dot(self.W_hy.T, dy) + dhnext # backprop into h. 
            dhraw = (1 - hs[t] * hs[t]) * dh # backprop through tanh nonlinearity #tanh'(x) = 1-tanh^2(x)
            dbh += dhraw
            dWxh += np.dot(dhraw, xs[t].T)
            dWhh += np.dot(dhraw, hs[t-1].T)
            dhnext = np.dot(self.W_hh.T, dhraw)
        
        for dparam in [dWxh, dWhh, dWhy, dbh, dby]: 
            np.clip(dparam, -5, 5, out=dparam) # clip to mitigate exploding gradients

        return dWxh, dWhh, dWhy, dbh, dby

    def train(self, article_text, hidden_size=500, n_iterations=1000, sequence_length=40, learning_rate=1e-1):
        self.article_text = article_text
        self.hidden_size = hidden_size
        chars = list(set(article_text))
        self.num_chars = len(chars)
        self.char_to_int = {c: i for i, c in enumerate(chars)}
        self.int_to_char = {i: c for i, c in enumerate(chars)}
        # original text encoded as integers, what we pass into our model
        self.integer_encoded = [self.char_to_int[i] for i in article_text]

        # Weights
        self.W_xh = np.random.randn(hidden_size, self.num_chars) * 0.01
        self.W_hh = np.random.randn(hidden_size, hidden_size) * 0.01
        self.W_hy = np.random.randn(self.num_chars, hidden_size) * 0.01
        # biases
        self.b_h = np.zeros((hidden_size, 1))
        self.b_y = np.zeros((self.num_chars, 1))
        # previous state
        self.h_prev  = np.zeros((hidden_size, 1)) # h_(t-1)
        
        batch_size = round((len(article_text) / sequence_length) + 0.5) # math.ceil
        data_pointer = 0

        # memory variables for Adagrad
        mWxh, mWhh, mWhy, mbh, mby = \
            [np.zeros_like(_) for _ in [self.W_xh, self.W_hh, self.W_hy, 
                                        self.b_h,  self.b_y]]

        for i in range(n_iterations):
            h_prev = np.zeros((hidden_size, 1)) # reset RNN memory
            data_pointer = 0 # go from start of data

            for b in range(batch_size):
                inputs = [self.char_to_int[ch] 
                          for ch in self.article_text[data_pointer:data_pointer+sequence_length]]
                targets = [self.char_to_int[ch] 
                           for ch in self.article_text[data_pointer+1:data_pointer+sequence_length+1]] # t+1        

                if (data_pointer+sequence_length+1 >= len(self.article_text) and b == batch_size-1): # processing of the last part of the input data. 
                    targets.append(self.char_to_int[" "])   # When the data doesn't fit, add space(" ") to the back.

                loss, ps, hs, xs = self.forward_prop(inputs, targets, h_prev)
                dWxh, dWhh, dWhy, dbh, dby = self.backward_prop(ps, inputs, hs, xs, targets) 

                # perform parameter update with Adagrad
                for param, dparam, mem in zip([self.W_xh, self.W_hh, self.W_hy, 
                                               self.b_h,  self.b_y], 
                                              [dWxh, dWhh, dWhy, dbh, dby], 
                                              [mWxh, mWhh, mWhy, mbh, mby]):
                    mem += dparam * dparam # elementwise
                    param += -learning_rate * dparam / np.sqrt(mem + 1e-8) # adagrad update      

                data_pointer += sequence_length # move data pointer

            if i % 100 == 0:
                print ('iter %d, loss: %f' % (i, loss)) # print progress
                
    def predict(self, test_char, length):
        x = np.zeros((self.num_chars, 1))
        x[self.char_to_int[test_char]] = 1
        ixes = []
        h = np.zeros((self.hidden_size, 1))

        for t in range(length):
            h = np.tanh(np.dot(self.W_xh, x) + np.dot(self.W_hh, h) + self.b_h)
            y = np.dot(self.W_hy, h) + self.b_y
            p = np.exp(y) / np.sum(np.exp(y))
            ix = np.random.choice(range(self.num_chars), p=p.ravel()) # ravel -> rank0
            # "ix" is a list of indexes selected according to the soft max probability.
            x = np.zeros((self.num_chars, 1)) # init
            x[ix] = 1 
            ixes.append(ix) # list
        
        txt = test_char + ''.join(self.int_to_char[i] for i in ixes)
        
        return txt
import requests

url = 'https://www.gutenberg.org/files/100/100-0.txt'
r = requests.get(url)

# subsample
start_i = 2965  # index of first text, THE SONNETS
length = 4000
article_text = r.text[start_i:start_i+length]
article_text = ' '.join(article_text.split())
model = RNN_NLP_Generator()
model.train(article_text)
model.predict(test_char='T', length=100)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11008125952260031, 'fraction_numerical': 0.011046216353478923, 'mean_word_length': 2.2190437270126684, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '16993', 'license': 'MIT', 'n_tokens_mistral': 2666, 'n_tokens_neox': 2496, 'n_words': 776, 'path': '16-Deep-Learning/01_RNN_and_LSTM_Assignment.ipynb', 'repo_name': 'shalevy1/data-science-journal', 'size': '7876'}