<jupyter_start><jupyter_text>Character-Based Neural Language Model<jupyter_code>try:
  %tensorflow_version 2.x
except Exception:
  pass
# Load the TensorBoard notebook extension
%load_ext tensorboard<jupyter_output><empty_output><jupyter_text>Install Dependencies<jupyter_code>!pip install tensorflow_text -q
!pip install wandb -q
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
import tensorflow_text as tftext
import tensorflow_datasets as tfds
import tensorflow_hub as hub
from tensorflow.keras.layers import Bidirectional, Concatenate, Dense, Dropout, Embedding, Input, InputLayer, Lambda, LSTM
from tensorflow.keras.layers import InputSpec, Layer
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.utils import plot_model
from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard
import datetime, os
import wandb
import re
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "No GPUs found."<jupyter_output><empty_output><jupyter_text>Tracking setup Login to your W&B account to track the model metrics and save progress.<jupyter_code>wandb.login()<jupyter_output><empty_output><jupyter_text>Initialize a new W&B run with:* *entity* — the team posting this run (your username)* *project* — the name of the project to which this run will belong<jupyter_code>wandb.init(entity = "dzlab", project = "tf-char-rnn")
# WandB – Config is a variable that holds and saves hyperparameters and inputs
config = wandb.config          # Initialize config
config.batch_size = 1024         #@param Input batch size for training
config.train_split = 0.8       #@param Train / Validation dataset split
config.seq_len = 40            #@param Length of character sequence to use for predicting the next character
config.embed_output = 100      #@param Ouput size of the Embedding layer
config.lstm_output_1 = 128     #@param Ouput size of 1st LSTM
config.lstm_output_2 = 128     #@param Ouput size of 2nd LSTM
batch_size, seq_len, train_split = 1024, 40, 0.8
embed_output, lstm_output_1, lstm_output_2 = 100, 128, 128<jupyter_output><empty_output><jupyter_text>DataElon Musk Tweets [Dataset](https://data.world/adamhelsinger/elon-musk-tweets-until-4-6-17), you can find more Twitter dataset [here](https://github.com/shaypal5/awesome-twitter-data).<jupyter_code># Make sure Pandas does not truncate columns in order to see all text
pd.set_option('display.max_colwidth', -1)
dataset = 'https://query.data.world/s/zo3rtfsdt5gfdy7k7hvas3e7ixudxp'
df = pd.read_csv(dataset)
df.head()
size = df.count()[0]<jupyter_output><empty_output><jupyter_text>Text Cleaning If you noticed, the text is encapsulated in a `b'...'`, we need to remove the starting token `b'` or `b"` and the endoing quotes `'` and `"`.<jupyter_code>df['text'] = df['text'].apply(lambda text: re.sub("^b'|^b\"|'$|\"$", '', text))
df.head()
!curl -O https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt<jupyter_output>% Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 1089k  100 1089k    0     0  2723k      0 --:--:-- --:--:-- --:--:-- 2716k<jupyter_text><jupyter_code>text = ''.join(open('input.txt', 'r').readlines()) # shakespear
text = '\n'.join(df['text']) # tweets
text = """Sing a song of sixpence, A pocket full of rye.
Four and twenty blackbirds, Baked in a pie.
When the pie was opened The birds began to sing; Wasn't that a dainty dish, To set before the king.
The king was in his counting house, Counting out his money;
The queen was in the parlour, Eating bread and honey.
The maid was in the garden, Hanging out the clothes, When down came a blackbird And pecked off her nose."""<jupyter_output><empty_output><jupyter_text>VocabularyCreate a character vocabulary<jupyter_code>text_tensor = tf.constant(text)
chars_raged_tensor = tf.strings.unicode_split(text_tensor, 'UTF-8')
chars = chars_raged_tensor.numpy()
vocab = list(set(sorted(chars)))
print(vocab)<jupyter_output>[b'\n', b'q', b'v', b'K', b'e', b'u', b':', b'N', b'w', b'E', b'U', b'd', b't', b'-', b'k', b'y', b'c', b's', b'h', b'G', b',', b'!', b'T', b' ', b'J', b'V', b'o', b'Z', b'P', b'j', b'W', b'l', b'I', b'r', b'g', b'X', b'p', b'A', b'a', b'C', b'&', b"'", b'M', b'z', b'L', b'f', b'H', b'O', b'R', b'F', b'.', b'3', b'S', b'?', b'b', b'x', b'B', b'Y', b'i', b'$', b'Q', b'n', b'm', b'D', b';']<jupyter_text>Build lookup tableCreate a token-index mapping to use when encode review text into something that can be processed by a neural network.<jupyter_code>def create_vocab_table(vocab, num_oov=1):
  """Create a lookup table for a vocabulary"""
  vocab_values = tf.range(tf.size(vocab, out_type=tf.int64), dtype=tf.int64)
  init = tf.lookup.KeyValueTensorInitializer(keys=vocab, values=vocab_values, key_dtype=tf.string, value_dtype=tf.int64)
  vocab_table = tf.lookup.StaticVocabularyTable(init, num_oov, lookup_key_dtype=tf.string)
  return vocab_table
vocab_table = create_vocab_table(vocab)
vocab_table.lookup(tf.constant("\n"))<jupyter_output><empty_output><jupyter_text>Create Dataset<jupyter_code>sequences = tftext.ngrams(chars_raged_tensor, width=seq_len+1, reduction_type=tftext.Reduction.STRING_JOIN, string_separator='')
sequences[:5]<jupyter_output><empty_output><jupyter_text>Expand the string sequences one dimension<jupyter_code>sequences = tf.expand_dims(sequences, -1)
train_size = int(len(sequences) * train_split)
valid_size = len(sequences) - train_size
train_ds = tf.data.Dataset.from_tensor_slices(sequences[:train_size])
valid_ds = tf.data.Dataset.from_tensor_slices(sequences[train_size:])<jupyter_output><empty_output><jupyter_text>PreprocessingCreate a preprocessing helper function which will be called on each sample in order to encode the tweets into a sequence of token indices.The output of this transformation is a RaggedTensor.<jupyter_code>@tf.function
def basic_preprocess(text):
  rt = tf.strings.unicode_split(text, 'UTF-8')
  rt = tf.ragged.map_flat_values(vocab_table.lookup, rt)
  features, labels = rt[..., :-1], rt[..., -1:]
  # in features, sequeeze the second from the last dimension as it was created by the spit
  features = tf.squeeze(features, axis=-2)
  # in labels, sequeeze the last dimension
  labels = tf.squeeze(labels, axis=-1)
  # make labels a regular tensor
  labels = labels.to_tensor()
  return features, labels
train_ds = train_ds.shuffle(train_size).batch(batch_size).map(basic_preprocess)
valid_ds = valid_ds.shuffle(valid_size).batch(batch_size).map(basic_preprocess)
train_examples_batch, train_labels_batch = next(iter(train_ds))
train_examples_batch.shape, train_labels_batch.shape
train_examples_batch[:2], train_labels_batch[:2]
train_examples_batch[0]
# list(map(lambda idx: vocab[idx], train_examples_batch[0].numpy()))<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>pad_val, vocab_size = len(vocab), len(vocab)+1<jupyter_output><empty_output><jupyter_text>W&B Callback<jupyter_code>wandb_callback = wandb.keras.WandbCallback(log_weights=True)<jupyter_output><empty_output><jupyter_text>Char-RNN How to Quickly Train a Text-Generating Neural Network for Free [article](https://minimaxir.com/2018/05/text-neural-networks/).  AttentionWeightedAverageAn attention layer on the ouput of previous RNN layers as well as the Embedding layer. The implentation is based on this [](https://gist.github.com/thomwolf/e309e779a08c1ba899514d44355cd6df)<jupyter_code>import tensorflow.keras.backend as K

class AttentionWeightedAverage(Layer):
    """
    Computes a weighted average of the different channels across timesteps.
    Uses 1 parameter pr. channel to compute the attention value for a single timestep.
    """

    def __init__(self, return_attention=False, **kwargs):
        self.init = tf.initializers.get('uniform')
        self.supports_masking = True
        self.return_attention = return_attention
        super(AttentionWeightedAverage, self).__init__(** kwargs)
        
    def get_config(self):
        config = super().get_config().copy()
        config.update({
            'return_attention': self.return_attention
        })
        return config

    def build(self, input_shape):
        self.input_spec = [InputSpec(ndim=3)]
        assert len(input_shape) == 3

        self.W = self.add_weight(shape=(input_shape[2], 1),
                                 name='{}_W'.format(self.name),
                                 initializer=self.init)
        #self.trainable_weights = [self.W]
        super(AttentionWeightedAverage, self).build(input_shape)

    def call(self, x, mask=None):
        # computes a probability distribution over the timesteps
        # uses 'max trick' for numerical stability
        # reshape is done to avoid issue with Tensorflow
        # and 1-dimensional weights
        logits = K.dot(x, self.W)
        x_shape = K.shape(x)
        logits = K.reshape(logits, (x_shape[0], x_shape[1]))
        ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))

        # masked timesteps have zero weight
        if mask is not None:
            mask = K.cast(mask, K.floatx())
            ai = ai * mask
        att_weights = ai / K.sum(ai, axis=1, keepdims=True)
        weighted_input = x * K.expand_dims(att_weights)
        result = K.sum(weighted_input, axis=1)
        if self.return_attention:
            return [result, att_weights]
        return result

    def get_output_shape_for(self, input_shape):
        return self.compute_output_shape(input_shape)

    def compute_output_shape(self, input_shape):
        output_len = input_shape[2]
        if self.return_attention:
            return [(input_shape[0], output_len), (input_shape[0], input_shape[1])]
        return (input_shape[0], output_len)

    def compute_mask(self, input, input_mask=None):
        if isinstance(input_mask, list):
            return [None] * len(input_mask)
        else:
            return None<jupyter_output><empty_output><jupyter_text>Build the model As our targets are integers, we use `sparse_categorical_crossentropy` loss function.<jupyter_code>input = Input(shape=(None,), dtype='int64', ragged=True)
dense = tftext.keras.layers.ToDense(pad_value=pad_val, mask=True)(input)
embed = Embedding(vocab_size, config.embed_output)(dense)
lstm1 = LSTM(config.lstm_output_1, return_sequences=True)(embed)
lstm2 = LSTM(config.lstm_output_2, return_sequences=True)(lstm1)
concat = Concatenate(axis=-1)([embed, lstm1, lstm2])
attn = AttentionWeightedAverage()(concat)
output = Dense(vocab_size, activation='softmax')(attn)
model = Model(inputs=[input], outputs=output)
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
model.summary()
model.fit(train_ds, validation_data=valid_ds, epochs=10, callbacks=[wandb_callback])
model.save(os.path.join(wandb.run.dir, "char-rnn.h5"))
plot_model(model, to_file='char-rnn.png', show_shapes=True)<jupyter_output><empty_output><jupyter_text>Simple RNN<jupyter_code>class WBCallback(tf.keras.callbacks.Callback):

  def on_epoch_end(self, epoch, logs=None):
    # logs is a dict containing the loss value, and all the metrics at the end of a batch or epoch.
    # We will simply log to W&B all the content of this dictionnary.
    wandb.log(logs)
model = tf.keras.Sequential([
  InputLayer(input_shape=(None,), dtype='int64', ragged=True),
  tftext.keras.layers.ToDense(pad_value=pad_val, mask=True),
  Lambda(lambda x:K.one_hot(K.cast(x,'int64'), vocab_size)),
  LSTM(lstm_output_1),
  Dense(vocab_size, activation='softmax')
])
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
model.summary()
model.fit(train_ds, validation_data=valid_ds, epochs=10, callbacks=[wandb_callback])
model.save(os.path.join(wandb.run.dir, "char-rnn-simple.h5"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12453300124533001, 'fraction_numerical': 0.013781652137816522, 'mean_word_length': 4.8818359375, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 68, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '527775', 'license': 'Unlicense', 'n_tokens_mistral': 4261, 'n_tokens_neox': 4095, 'n_words': 985, 'path': 'tensorflow/Character_Based_Neural_Language_Model.ipynb', 'repo_name': 'dzlab/deepprojects', 'size': '12045'}