<jupyter_start><jupyter_text>CHARACTER LEVEL GENERATIONIn this notebook we build a character level RNN, using Gated Recurrent Networks (GRU).For a subject matter we chose President Trump's tweets, because we felt his style was noticeable. We also train our model on Shakespeare's corpus of text. With this project we wanted two deliverables.     1. To create our own RNN text generation model using either LSTM or GRU frameworks.    2. To build a webpage that would display a real tweet next to a fake one, allowing the user to guess which one was real.    This notebook is focused on building the model.<jupyter_code>import pandas as pd
import numpy as np
import tensorflow as tf
import random
import sys
import pickle
import csv
import os
import matplotlib.pyplot as plt
import re
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, RNN, Softmax, Flatten, Dropout, Input
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras.callbacks import LambdaCallback
from tensorflow.keras.models import load_model
from sklearn.model_selection import train_test_split

tf.enable_eager_execution()
# data = pd.read_csv("../Load_Tweets/data/tweet_data.csv") # this will break if this file is moved!
# data.head()

data = pd.read_csv("../Load_Tweets/data/Full Trump Archive.csv")
data.head()
# data['TEXT'][100]

data['text'][100]
data['text'].dtypes

# data['TEXT'].apply(lambda x: len(x)).describe()

data['text'].count()
# Put all the tweets into one string

# tweet_txt = data['TEXT'][:].str.cat(sep=' ')
tweet_txt = data['text'][:].str.cat(sep=' ')
print('{} : total characters in our dataset'.format(len(tweet_txt)))

# Get all the unique characters used, and make a character mapping. 
# Here we set Global Variables that are used throughout the code.

# with open('../Load_Tweets/data/ArtOfTheDeal.txt') as f:
#     book_txt = f.read()
    
# tweet_txt = tweet_txt + book_txt
# path_to_file = tf.keras.utils.get_file(
#     'shakespeare.txt', 
#     'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')

# # Read, then decode for py2 compat.
# tweet_txt = open(path_to_file, 'rb').read().decode(encoding='utf-8')

chars = list(set(tweet_txt))
chars.sort()
char_to_index = dict((c, i) for i, c in enumerate(chars))
index_to_char = np.array(chars)
print("Number of unique characters: ", len(chars))
maxlen = 30 # 141 Chosen because the average length of a tweet in our data is 141 characters.

tweet_int = np.array([char_to_index[char] for char in tweet_txt])
tweet_int[:20]
seq_length = 100
examples_per_epoch = len(tweet_txt)//seq_length
char_dataset = tf.data.Dataset.from_tensor_slices(tweet_int)

for i in char_dataset.take(5):
    print(index_to_char[i.numpy()])
sequences = char_dataset.batch(seq_length+1, drop_remainder=True)

for item in sequences.take(5):
    print(repr(''.join(index_to_char[item.numpy()])))
# Here we actual build the data.

def split_input_target(chunk):
    input_text = chunk[:-1]
    target_text = chunk[1:]
    return input_text, target_text

dataset = sequences.map(split_input_target)
for input_example, target_example in  dataset.take(1):
    print ('Input data: ', repr(''.join(index_to_char[input_example.numpy()])))
    print ('Target data:', repr(''.join(index_to_char[target_example.numpy()])))
# Batch size 
BATCH_SIZE = 64
steps_per_epoch = examples_per_epoch//BATCH_SIZE

# Buffer size
BUFFER_SIZE = 10000

dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)

dataset
# Here is a model using the Keras Functional Api.
if tf.test.is_gpu_available():
    rnn = tf.keras.layers.CuDNNGRU
    print("We are on the GPU!!!")
else:
    import functools
    rnn = functools.partial(
        tf.keras.layers.GRU, recurrent_activation='sigmoid')
#     tf.keras.layers.LSTM
    
def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
    model = tf.keras.Sequential([
        tf.keras.layers.Embedding(vocab_size, embedding_dim, 
                              batch_input_shape=[batch_size, None]),
        rnn(rnn_units,
            return_sequences=True, 
            recurrent_initializer='glorot_uniform',
#             bias_regularizer=tf.keras.regularizers.l1(l=0.01),
            stateful=True),
        tf.keras.layers.Dropout(rate=0.2, noise_shape=(batch_size, 1, rnn_units)),
        rnn(rnn_units,
            return_sequences=True, 
            recurrent_initializer='glorot_uniform',
#             bias_regularizer=tf.keras.regularizers.l1(l=0.01),
            stateful=True),
        tf.keras.layers.Dense(vocab_size)
    ])
    
    return model

vocab_size = len(chars)
embedding_dim = 256
rnn_units = 1024
batch_size=BATCH_SIZE



model = build_model(
  vocab_size = vocab_size, 
  embedding_dim=embedding_dim, 
  rnn_units=rnn_units, 
  batch_size=batch_size)

model.summary()
def loss(labels, logits):
    return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)

model.compile(
    optimizer = tf.train.AdamOptimizer(),
    loss = loss
)
# Directory where the checkpoints will be saved
checkpoint_dir = './training_checkpoints'
# Name of the checkpoint files
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt_{epoch}")

checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_prefix,
    save_weights_only=True)

EPOCHS = 30

early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=5, restore_best_weights=True)

history = model.fit(
    dataset.repeat(),
    validation_data=dataset,
    validation_steps=30,
    epochs=EPOCHS, 
    steps_per_epoch=steps_per_epoch, 
    callbacks=[checkpoint_callback])

# model tests based on 15 epochs.

# First try model: 3 GRU layers, 2 dropouts, third GRU has 2^9 nodes, down from 2^10 in between.
# Loss: 1.1043

# Second try model: 2 GRU layers, 1 dropout in between.
# Loss: 0.9172 on 50 epochs early stopped at 38 with a loss of 0.8059

# Third try model: 3 GRU layers, 2 dropouts, in between.
# Loss: 1.0251

# First try model: 3 GRU layers, 1 dropouts, third GRU has 2^9 nodes, down from 2^10 in between.
# Loss: 0.9321 after 50 epochs we got a loss of 0.8640






model_g = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)

model_g.load_weights(tf.train.latest_checkpoint(checkpoint_dir))

model_g.build(tf.TensorShape([1, None]))

model_g.summary()
model_g = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)

model_g.load_weights('../Saved_models/third_model_weights.h5')

model_g.build(tf.TensorShape([1, None]))

model_g.summary()
def generate_text(model, start_string, length):
    # Evaluation step (generating text using the learned model)

    # Number of characters to generate
    num_generate = length

    # Converting our start string to numbers (vectorizing) 
    input_eval = [char_to_index[s] for s in start_string]
    input_eval = tf.expand_dims(input_eval, 0)

    # Empty string to store our results
    text_generated = []

    # Low temperatures results in more predictable text.
    # Higher temperatures results in more surprising text.
    # Experiment to find the best setting.
    temperature = 0.95

    # Here batch size == 1
    model.reset_states()
    for i in range(num_generate):
        predictions = model(input_eval)
        # remove the batch dimension
        predictions = tf.squeeze(predictions, 0)

        # using a multinomial distribution to predict the word returned by the model
        predictions = predictions / temperature
        predicted_id = tf.multinomial(predictions, num_samples=1)[-1,0].numpy()

        # We pass the predicted word as the next input to the model
        # along with the previous hidden state
        input_eval = tf.expand_dims([predicted_id], 0)

        text_generated.append(index_to_char[predicted_id])

    return (start_string + ''.join(text_generated))
l = 500

print(generate_text(model_g, start_string=u" ", length=l))
l = 200

print(generate_text(model_g, 
                    start_string=og_tweets['TEXT'][14][:30].replace("\n", ""), 
                    length=l))<jupyter_output>Wow. Unbelievable. https://t.co/1ELocVhmXL Thank you it or Give Up -- "on recognized Club today.  Young story with four new Governor of Florida. She got a big different president in a long way to thank about! @Jernmiletteconsen: A<jupyter_text>""" HERE I AM GENERATING A LIST OF TWEETS THAT END APPROPRIATELY. """<jupyter_code>og_tweets = pd.read_csv("../Load_Tweets/data/original_tweets.csv")
og_tweets.head()
og_tweets['TEXT'][0][:100].replace("\n", "")
# make fake tweets


# make the fake tweets look good
# store everything and pickle it
# 
# Here we save the model

model.save('../Saved_models/third_model.h5')
model.save_weights('../Saved_models/third_model_weights.h5')
""" HERE I AM DOING SOME MODEL TESTING """
model = load_model('../Saved_models/first_char_model.h5')
cross_entropy_loss, accuracy = model.evaluate(X, y, batch_size=128)
df = pd.read_csv('../Load_Tweets/data/original_tweets.csv')
# building out support for real URLS, also need to updat model.data to accomidate new model.


link_txt = df['TEXT'][:].str.cat(sep=' ')
re.findall("(https\S*)", link_txt)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1085590465872156, 'fraction_numerical': 0.016901408450704224, 'mean_word_length': 4.203494926719278, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '488308', 'license': 'MIT', 'n_tokens_mistral': 3217, 'n_tokens_neox': 3044, 'n_words': 809, 'path': 'Model/Jackson_model-2.ipynb', 'repo_name': 'EthanH43/text_generator', 'size': '9230'}