<jupyter_start><jupyter_text>Training an RNN with Text Data  Load Libraries<jupyter_code>import urllib2
import PIL
import os
import numpy as np
import math
import PIL.Image as Image
import pandas as pd
import re
import matplotlib.pyplot as plt

'''Trains a LSTM on the IMDB sentiment classification task.
The dataset is actually too small for LSTM to be of any advantage
compared to simpler, much faster methods such as TF-IDF + LogReg.
Notes:
- RNNs are tricky. Choice of batch size is important,
choice of loss and optimizer is critical, etc.
Some configurations won't converge.
- LSTM loss decrease patterns during training can be quite different
from what you see with CNNs/MLPs/etc.
'''
from __future__ import print_function

from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Embedding
from keras.layers import LSTM
from keras.datasets import imdb
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.callbacks import TensorBoard<jupyter_output>Using TensorFlow backend.<jupyter_text>Set up Text Cleaner function<jupyter_code># Set up helper cleaner function
def cleaner(cell):
    line = str(cell)
    line = line.replace('[u', '').replace(']', '').replace(',', '').replace("u'", '').replace("'", '')
    line = re.sub("(^|\W)\d+($|\W)", " ", line)
    line = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", line)
    line = re.sub(r"\'s", " \'s", line)
    line = re.sub(r"\'ve", " \'ve", line)
    line = re.sub(r"n\'t", " n\'t", line)
    line = re.sub(r"\'re", " \'re", line)
    line = re.sub(r"\'d", " \'d", line)
    line = re.sub(r"\'ll", " \'ll", line)
    line = re.sub(r",", " , ", line)
    line = re.sub(r"!", " ! ", line)
    line = re.sub(r"\(", " \( ", line)
    line = re.sub(r"\)", " \) ", line)
    line = re.sub(r"\?", " \? ", line)
    line = re.sub(r"\s{2,}", " ", line)
    return line<jupyter_output><empty_output><jupyter_text>Prepare and Clean Data<jupyter_code># load train
train = pd.read_csv("milestone_4/train_full.csv")

# drop rogue column
train.drop("Unnamed: 0", axis=1, inplace=True)

# alias a copy
curr_df = train.copy()

# Store target labels
string_y = curr_df.ix[:,0:19] 

# create matrix of data's string values
string_cols = ["director", "lead actors", "overview", "title"]
string_matrix = curr_df[string_cols]

# trim trailing and leading spaces
string_matrix = string_matrix.apply(lambda col: col.str.strip())

# combine text columns into one
total = string_matrix['director'] + " " + string_matrix['lead actors'] + " " + string_matrix['overview'] + " " + string_matrix['title']

# clean up and save
total = total.apply(cleaner)
total = total.apply(lambda x: ''.join([" " if ord(i) < 32 or ord(i) > 126 else i for i in x]))
text_train = pd.DataFrame(total)<jupyter_output><empty_output><jupyter_text>Preprocess Text Data<jupyter_code># set global params
max_features = 2000
maxlen = 1000 

# tokenize data
tokenizer = Tokenizer(num_words=1000)
tokenizer.fit_on_texts(text_train.ix[:].values.ravel())
sequences = tokenizer.texts_to_sequences(text_train.ix[:].values.ravel() )
word_index = tokenizer.word_index

print('Found %s unique tokens.' % len(word_index))

# pad data to be standard
data = pad_sequences(sequences, maxlen=800)

print('Shape of data tensor:', data.shape)<jupyter_output>Found 28518 unique tokens.
Shape of data tensor: (7220, 800)<jupyter_text>Consolidate targets for one model<jupyter_code># simplify labels for training
labels = pd.DataFrame() 
labels["other"] = ""
labels["heartbeat"] = ""
labels["lighthearted"] = ""

for index, row in string_y.iterrows():
    if row["36"] == 1 or row["37"] == 1 or row["10770"] == 1:
        labels.set_value(index, "other", 1)
    else:
        labels.set_value(index, "other", 0)
        
    if row["12"] == 1 or row["14"] == 1 or row["18"] == 1 or row["27"] == 1 or row["28"] == 1 or row["53"] == 1 or row["80"] == 1 or row["10749"] == 1 or row["10752"] == 1:
        labels.set_value(index, "heartbeat", 1)
    else:
        labels.set_value(index, "heartbeat", 0)
        
    if row["16"] == 1 or row["35"] == 1 or row["10402"] == 1 or row["10751"] == 1:
        labels.set_value(index, "lighthearted", 1)
    else:
        labels.set_value(index, "lighthearted", 0)
# also store the 19 labels to compare
old_labels = string_y
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)<jupyter_output>Shape of data tensor: (7220, 800)
Shape of label tensor: (7220, 3)<jupyter_text>Create and format train and test data<jupyter_code># split the data into a training set and a validation set
indices = np.arange(data.shape[0])
np.random.shuffle(indices)

#sub select
data = data[indices]

# prep labels
labels = labels.values
labels = labels[indices]

# prep old labels
old_labels = old_labels.values
old_labels = old_labels[indices]

# format data and split
nb_validation_samples = int(.3 * data.shape[0])
x_train = data[:-nb_validation_samples]
y_train = labels[:-nb_validation_samples]
x_val = data[-nb_validation_samples:]
y_val = labels[-nb_validation_samples:]

# format old labels and split
y_train_old = old_labels[:-nb_validation_samples]
y_val_old = old_labels[-nb_validation_samples:]

print(x_train.shape)
print(y_train.shape)
print(y_train_old.shape)
print(y_val_old.shape)<jupyter_output>(5054, 800)
(5054, 3)
(5054, 19)
(2166, 19)<jupyter_text>Build Models  Here we will compare the accuracy of the RNN predicting the three label description of movies along with the full 19.  Set up Model Plotting Function<jupyter_code>def plot_acc(model_history):
    plt.figure(figsize=(8, 6))

    max_train_acc = round(max(model_history.history['acc']), 2)
    max_test_acc = round(max(model_history.history['val_acc']), 2)

    plt.plot(model_history.history['acc'], 
             label="Train Acc (Max = {0})".format(max_train_acc))
    plt.plot(model_history.history['val_acc'], 
             label="Test Acc (Max = {0})".format(max_test_acc))

    plt.title("Model accuracy as a function of epochs")
    plt.xlabel("epoch")
    plt.legend(loc="lower right")
    plt.ylabel("accuracy")
    plt.show()
print('Build model with 3 data targets...')
model_3 = Sequential()
model_3.add(Embedding(max_features, 400))
model_3.add(LSTM(3, dropout=0.2, recurrent_dropout=0.2))

# try using different optimizers and different optimizer configs
model_3.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
model_3.summary()
print('Build model with 19 data targets...')
model_19 = Sequential()
model_19.add(Embedding(max_features, 400))
model_19.add(LSTM(19, dropout=0.2, recurrent_dropout=0.2))

# try using different optimizers and different optimizer configs
model_19.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
model_19.summary()<jupyter_output>Build model with 19 data targets...
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_2 (Embedding)      (None, None, 400)         800000    
_________________________________________________________________
lstm_2 (LSTM)                (None, 19)                31920     
=================================================================
Total params: 831,920.0
Trainable params: 831,920
Non-trainable params: 0.0
_________________________________________________________________<jupyter_text>Training model with 3 label targets<jupyter_code># happy learning!
history_3 = model_3.fit(x_train, y_train,
                    batch_size=361,
                    epochs=20,
                    validation_data=(x_val, y_val))
plot_acc(history_3)<jupyter_output>Train on 5054 samples, validate on 2166 samples
Epoch 1/20
5054/5054 [==============================] - 21s - loss: 0.4572 - acc: 0.7876 - val_loss: 0.5135 - val_acc: 0.7813[...]<jupyter_text>Training model with 19 label targets<jupyter_code># happy learning!
history_19 = model_19.fit(x_train, y_train_old,
                    batch_size=361,
                    epochs=20,
                    validation_data=(x_val, y_val_old))
plot_acc(history_19)<jupyter_output>Train on 5054 samples, validate on 2166 samples
Epoch 1/20
5054/5054 [==============================] - 22s - loss: 1.0733 - acc: 0.8404 - val_loss: 0.7650 - val_acc: 0.8382[...]
{'fraction_non_alphanumeric': 0.2938852118937074, 'fraction_numerical': 0.032994171688234715, 'mean_word_length': 5.315658140985652, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '116677', 'license': 'Apache-2.0', 'n_tokens_mistral': 4688, 'n_tokens_neox': 4398, 'n_words': 734, 'path': 'RNNonText.ipynb', 'repo_name': 'noahyonack/cs109b_final', 'size': '10123'}