<jupyter_start><jupyter_text>Recipe Generation using Recurrent Neural Network (RNN)  Importing dependencies<jupyter_code>import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
import platform
import time
import pathlib
import os
import json

print('Python version:', platform.python_version())
print('Tensorflow version:', tf.__version__)
print('Keras version:', tf.keras.__version__)<jupyter_output>Python version: 3.6.9
Tensorflow version: 2.2.0
Keras version: 2.3.0-tf<jupyter_text>Exploring datasets- 🤷 [Recipe Ingredients Dataset](https://www.kaggle.com/kaggle/recipe-ingredients-dataset/home) _(doesn't have ingredients proportions)_- 🤷 [Recipe1M+](http://pic2recipe.csail.mit.edu/) _(requires registration to download)_- 🤷 [Epicurious - Recipes with Rating and Nutrition](https://www.kaggle.com/hugodarwood/epirecipes?select=full_format_recipes.json) _(~20k recipes only, it would be nice to find more)_- 👍🏻 [**Recipe box**](https://eightportions.com/datasets/Recipes/) _(~125,000 recipes with ingredients proportions)_  Loading the dataset<jupyter_code># Create cache folder.
cache_dir = './tmp'
pathlib.Path(cache_dir).mkdir(exist_ok=True)
# Download and unpack the dataset.
dataset_file_name = 'recipes_raw.zip'
dataset_file_origin = 'https://storage.googleapis.com/recipe-box/recipes_raw.zip'

dataset_file_path = tf.keras.utils.get_file(
    fname=dataset_file_name,
    origin=dataset_file_origin,
    cache_dir=cache_dir,
    extract=True,
    archive_format='zip'
)

print(dataset_file_path)
!ls -la ./tmp/datasets/
def load_dataset(silent=False):
    dataset_file_names = [
        'recipes_raw_nosource_ar.json',
        'recipes_raw_nosource_epi.json',
        'recipes_raw_nosource_fn.json',
    ]
    
    dataset = []

    for dataset_file_name in dataset_file_names:
        dataset_file_path = f'{cache_dir}/datasets/{dataset_file_name}'

        with open(dataset_file_path) as dataset_file:
            json_data_dict = json.load(dataset_file)
            json_data_list = list(json_data_dict.values())
            dict_keys = [key for key in json_data_list[0]]
            dict_keys.sort()
            dataset += json_data_list

            if silent == False:
                print(dataset_file_path)
                print('===========================================')
                print('Number of examples: ', len(json_data_list), '\n')
                print('Example object keys:\n', dict_keys, '\n')
                print('Example object:\n', json_data_list[0], '\n')
                print('Required keys:\n')
                print('  title: ', json_data_list[0]['title'], '\n')
                print('  ingredients: ', json_data_list[0]['ingredients'], '\n')
                print('  instructions: ', json_data_list[0]['instructions'])
                print('\n\n')
    
    return dataset  
dataset_raw = load_dataset()  
print('Total number of raw examples: ', len(dataset_raw))<jupyter_output>Total number of raw examples:  125164<jupyter_text>Preprocessing the dataset  Filtering out incomplete examples<jupyter_code>def recipe_validate_required_fields(recipe):
    required_keys = ['title', 'ingredients', 'instructions']
    
    if not recipe:
        return False
    
    for required_key in required_keys:
        if not recipe[required_key]:
            return False
        
        if type(recipe[required_key]) == list and len(recipe[required_key]) == 0:
            return False
    
    return True
dataset_validated = [recipe for recipe in dataset_raw if recipe_validate_required_fields(recipe)]

print('Dataset size BEFORE validation', len(dataset_raw))
print('Dataset size AFTER validation', len(dataset_validated))
print('Number of invalide recipes', len(dataset_raw) - len(dataset_validated))<jupyter_output>Dataset size BEFORE validation 125164
Dataset size AFTER validation 122938
Number of invalide recipes 2226<jupyter_text>Converting recipes objects into strings<jupyter_code>STOP_WORD_TITLE = '[title]\n\n'
STOP_WORD_INGREDIENTS = '\n[ingredients]\n\n'
STOP_WORD_INSTRUCTIONS = '\n[instructions]\n\n'
def recipe_to_string(recipe):
    noize_string = 'ADVERTISEMENT'
    
    title = recipe['title']
    ingredients = recipe['ingredients']
    instructions = recipe['instructions'].split('\n')
    
    ingredients_string = ''
    for ingredient in ingredients:
        ingredient = ingredient.replace(noize_string, '')
        if ingredient:
            ingredients_string += f'• {ingredient}\n'
    
    instructions_string = ''
    for instruction in instructions:
        instruction = instruction.replace(noize_string, '')
        if instruction:
            instructions_string += f'▪︎ {instruction}\n'
    
    return f'{STOP_WORD_TITLE}{title}\n{STOP_WORD_INGREDIENTS}{ingredients_string}{STOP_WORD_INSTRUCTIONS}{instructions_string}'
dataset_stringified = [recipe_to_string(recipe) for recipe in dataset_validated]

print('Stringified dataset size: ', len(dataset_stringified))
for recipe_index, recipe_string in enumerate(dataset_stringified[:10]):
    print('Recipe #{}\n---------'.format(recipe_index + 1))
    print(recipe_string)
    print('\n')
print(dataset_stringified[50000])<jupyter_output>[title]

Herbed Bean Ragoût 

[ingredients]

• 6 ounces haricots verts (French thin green beans), trimmed and halved crosswise
• 1 (1-pound) bag frozen edamame (soybeans in the pod) or 1 1/4 cups frozen shelled edamame, not thawed
• 2/3 cup finely chopped onion
• 2 garlic cloves, minced
• 1 Turkish bay leaf or 1/2 California bay leaf
• 2 (3-inch) fresh rosemary sprigs
• 1/2 teaspoon salt
• 1/4 teaspoon black pepper
• 1 tablespoon olive oil
• 1 medium carrot, cut into 1/8-inch dice
• 1 medium celery rib, cut into 1/8-inch dice
• 1 (15- to 16-ounces) can small white beans, rinsed and drained
• 1 1/2 cups chicken stock or low-sodium broth
• 2 tablespoons unsalted butter
• 2 tablespoons finely chopped fresh flat-leaf parsley
• 1 tablespoon finely chopped fresh chervil (optional)
• Garnish: fresh chervil sprigs

[instructions]

▪︎ Cook haricots verts in a large pot of boiling salted water until just tender, 3 to 4 minutes. Transfer with a slotted spoon to a bowl of ice and cold water, then [...]<jupyter_text>Filtering out large receipts<jupyter_code>recipes_lengths = []
for recipe_text in dataset_stringified:
    recipes_lengths.append(len(recipe_text))
plt.hist(recipes_lengths, bins=50)
plt.show()
plt.hist(recipes_lengths, range=(0, 8000), bins=50)
plt.show()
MAX_RECIPE_LENGTH = 2000
def filter_recipes_by_length(recipe_test):
  return len(recipe_test) <= MAX_RECIPE_LENGTH 
dataset_filtered = [recipe_text for recipe_text in dataset_stringified if filter_recipes_by_length(recipe_text)]

print('Dataset size BEFORE filtering: ', len(dataset_stringified))
print('Dataset size AFTER filtering: ', len(dataset_filtered))
print('Number of etiminated recipes: ', len(dataset_stringified) - len(dataset_filtered))<jupyter_output>Dataset size BEFORE filtering:  122938
Dataset size AFTER filtering:  99377
Number of etiminated recipes:  23561<jupyter_text>Summarizing dataset parameters<jupyter_code>TOTAL_RECIPES_NUM = len(dataset_filtered)

print('MAX_RECIPE_LENGTH: ', MAX_RECIPE_LENGTH)
print('TOTAL_RECIPES_NUM: ', TOTAL_RECIPES_NUM)<jupyter_output>MAX_RECIPE_LENGTH:  2000
TOTAL_RECIPES_NUM:  99377<jupyter_text>Creating vocabulary<jupyter_code>STOP_SIGN = '␣'
tokenizer = tf.keras.preprocessing.text.Tokenizer(
    char_level=True,
    filters='',
    lower=True,
    split=''
)
tokenizer.fit_on_texts([STOP_SIGN])
tokenizer.fit_on_texts(dataset_filtered)
VOCABULARY_SIZE = len(tokenizer.word_counts)

print('VOCABULARY_SIZE: ', VOCABULARY_SIZE)
tokenizer.get_config()
print(tokenizer.index_word[50])
print(tokenizer.index_word[100])
tokenizer.word_index['ö']<jupyter_output><empty_output><jupyter_text>Vectorizing the dataset<jupyter_code>def recipe_sequence_to_string(recipe_sequence):
    recipe_stringified = tokenizer.sequences_to_texts([recipe_sequence])[0]
    recipe_stringified = recipe_stringified.replace('   ', '_').replace(' ', '').replace('_', ' ')
    print(recipe_stringified)
dataset_vectorized = tokenizer.texts_to_sequences(dataset_filtered)
print('Vectorized dataset size', len(dataset_vectorized))
print(dataset_vectorized[0][:20], '...')
recipe_sequence_to_string(dataset_vectorized[0])<jupyter_output>[title]

slow cooker chicken and dumplings

[ingredients]

• 4 skinless, boneless chicken breast halves 
• 2 tablespoons butter 
• 2 (10.75 ounce) cans condensed cream of chicken soup 
• 1 onion, finely diced 
• 2 (10 ounce) packages refrigerated biscuit dough, torn into pieces 

[instructions]

▪︎ place the chicken, butter, soup, and onion in a slow cooker, and fill with enough water to cover.
▪︎ cover, and cook for 5 to 6 hours on high. about 30 minutes before serving, place the torn biscuit dough in the slow cooker. cook until the dough is no longer raw in the center.<jupyter_text>Add padding to sequences<jupyter_code>for recipe_index, recipe in enumerate(dataset_vectorized[:10]):
    print('Recipe #{} length: {}'.format(recipe_index, len(recipe)))
dataset_vectorized_padded_without_stops = tf.keras.preprocessing.sequence.pad_sequences(
    dataset_vectorized,
    padding='post',
    truncating='post',
    # We use -1 here and +1 in the next step to make sure that all recipes will have at least 1 stops
    # sign at the end, since each sequence will be shifted and truncated afterwards (to generate X and Y sequences).
    maxlen=MAX_RECIPE_LENGTH-1,
    value=tokenizer.texts_to_sequences([STOP_SIGN])[0]
)
dataset_vectorized_padded = tf.keras.preprocessing.sequence.pad_sequences(
    dataset_vectorized_padded_without_stops,
    padding='post',
    truncating='post',
    maxlen=MAX_RECIPE_LENGTH+1,
    value=tokenizer.texts_to_sequences([STOP_SIGN])[0]
)
for recipe_index, recipe in enumerate(dataset_vectorized_padded[:10]):
    print('Recipe #{} length: {}'.format(recipe_index, len(recipe)))
recipe_sequence_to_string(dataset_vectorized_padded[0])<jupyter_output>[title]

slow cooker chicken and dumplings

[ingredients]

• 4 skinless, boneless chicken breast halves 
• 2 tablespoons butter 
• 2 (10.75 ounce) cans condensed cream of chicken soup 
• 1 onion, finely diced 
• 2 (10 ounce) packages refrigerated biscuit dough, torn into pieces 

[instructions]

▪︎ place the chicken, butter, soup, and onion in a slow cooker, and fill with enough water to cover.
▪︎ cover, and cook for 5 to 6 hours on high. about 30 minutes before serving, place the torn biscuit dough in the slow cooker. cook until the dough is no longer raw in the center.
␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣[...]<jupyter_text>Create TensorFlow dataset<jupyter_code>dataset = tf.data.Dataset.from_tensor_slices(dataset_vectorized_padded)

print(dataset)
for recipe in dataset.take(1):
    print('Raw recipe:\n', recipe.numpy(), '\n\n\n')
    print('Stringified recipe:\n')
    recipe_sequence_to_string(recipe.numpy())<jupyter_output>Raw recipe:
 [ 34   3   7 ... 129 129 129] 



Stringified recipe:

[title]

slow cooker chicken and dumplings

[ingredients]

• 4 skinless, boneless chicken breast halves 
• 2 tablespoons butter 
• 2 (10.75 ounce) cans condensed cream of chicken soup 
• 1 onion, finely diced 
• 2 (10 ounce) packages refrigerated biscuit dough, torn into pieces 

[instructions]

▪︎ place the chicken, butter, soup, and onion in a slow cooker, and fill with enough water to cover.
▪︎ cover, and cook for 5 to 6 hours on high. about 30 minutes before serving, place the torn biscuit dough in the slow cooker. cook until the dough is no longer raw in the center.
␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣␣[...]<jupyter_text>Split examples on `input` and `target` texts For each sequence, duplicate and shift it to form the input and target text. For example, say sequence_length is 4 and our text is `Hello`. The input sequence would be `Hell`, and the target sequence `ello`.<jupyter_code>def split_input_target(recipe):
    input_text = recipe[:-1]
    target_text = recipe[1:]
    
    return input_text, target_text
dataset_targeted = dataset.map(split_input_target)

print(dataset_targeted)
for input_example, target_example in dataset_targeted.take(1):
    print('Input sequence size:', repr(len(input_example.numpy())))
    print('Target sequence size:', repr(len(target_example.numpy())))
    print()
    
    input_stringified = tokenizer.sequences_to_texts([input_example.numpy()[:50]])[0]
    target_stringified = tokenizer.sequences_to_texts([target_example.numpy()[:50]])[0]
    
    print('Input:  ', repr(''.join(input_stringified)))
    print('Target: ', repr(''.join(target_stringified)))<jupyter_output>Input sequence size: 2000
Target sequence size: 2000

Input:   '[ t i t l e ] \n \n s l o w   c o o k e r   c h i c k e n   a n d   d u m p l i n g s \n \n [ i n g r e'
Target:  't i t l e ] \n \n s l o w   c o o k e r   c h i c k e n   a n d   d u m p l i n g s \n \n [ i n g r e d'<jupyter_text>Each index of these vectors are processed as one time step. For the input at time step 0, the model receives the index for `[` and tries to predict the index for `T` as the next character. At the next time-step, it does the same thing but the RNN considers the previous step context in addition to the current input character.<jupyter_code>for i, (input_idx, target_idx) in enumerate(zip(input_example[:10], target_example[:10])):
    print('Step {:2d}'.format(i))
    print('  input: {} ({:s})'.format(input_idx, repr(tokenizer.sequences_to_texts([[input_idx.numpy()]])[0])))
    print('  expected output: {} ({:s})'.format(target_idx, repr(tokenizer.sequences_to_texts([[target_idx.numpy()]])[0])))<jupyter_output>Step  0
  input: 34 ('[')
  expected output: 3 ('t')
Step  1
  input: 3 ('t')
  expected output: 7 ('i')
Step  2
  input: 7 ('i')
  expected output: 3 ('t')
Step  3
  input: 3 ('t')
  expected output: 10 ('l')
Step  4
  input: 10 ('l')
  expected output: 2 ('e')
Step  5
  input: 2 ('e')
  expected output: 33 (']')
Step  6
  input: 33 (']')
  expected output: 16 ('\n')
Step  7
  input: 16 ('\n')
  expected output: 16 ('\n')
Step  8
  input: 16 ('\n')
  expected output: 8 ('s')
Step  9
  input: 8 ('s')
  expected output: 10 ('l')<jupyter_text>Split up the dataset into batches<jupyter_code>print(dataset_targeted)
print('TOTAL_RECIPES_NUM: ', TOTAL_RECIPES_NUM)
print('MAX_RECIPE_LENGTH: ', MAX_RECIPE_LENGTH)
print('VOCABULARY_SIZE: ', VOCABULARY_SIZE)
# Batch size.
BATCH_SIZE = 128

# Buffer size to shuffle the dataset (TF data is designed to work
# with possibly infinite sequences, so it doesn't attempt to shuffle
# the entire sequence in memory. Instead, it maintains a buffer in
# which it shuffles elements).
SHUFFLE_BUFFER_SIZE = 1000
dataset_train = dataset_targeted \
    .shuffle(SHUFFLE_BUFFER_SIZE) \
    .batch(BATCH_SIZE, drop_remainder=True) \
    .repeat()

print(dataset_train)
for input_text, target_text in dataset_train.take(1):
    print('1st batch: input_text:', input_text)
    print()
    print('1st batch: target_text:', target_text)<jupyter_output>1st batch: input_text: tf.Tensor(
[[ 34   3   7 ... 129 129 129]
 [ 34   3   7 ... 129 129 129]
 [ 34   3   7 ... 129 129 129]
 ...
 [ 34   3   7 ... 129 129 129]
 [ 34   3   7 ... 129 129 129]
 [ 34   3   7 ... 129 129 129]], shape=(128, 2000), dtype=int32)

1st batch: target_text: tf.Tensor(
[[  3   7   3 ... 129 129 129]
 [  3   7   3 ... 129 129 129]
 [  3   7   3 ... 129 129 129]
 ...
 [  3   7   3 ... 129 129 129]
 [  3   7   3 ... 129 129 129]
 [  3   7   3 ... 129 129 129]], shape=(128, 2000), dtype=int32)<jupyter_text>Build the model Use [tf.keras.Sequential](https://www.tensorflow.org/api_docs/python/tf/keras/Sequential) to define the model. For this experiment we will use the following layer types:- [tf.keras.layers.Embedding](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Embedding): The input layer. A trainable lookup table that will map the numbers of each character to a vector with `embedding_dim` dimensions;- [tf.keras.layers.LSTM](https://www.tensorflow.org/api_docs/python/tf/keras/layers/LSTM): A type of RNN with size units=rnn_units (You can also use a GRU layer here.)- [tf.keras.layers.Dense](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense): The output layer, with vocab_size outputs.<jupyter_code># Let's do a quick detour and see how Embeding layer works.
# It takes several char indices sequences (batch) as an input.
# It encodes every character of every sequence to a vector of tmp_embeding_size length.
tmp_vocab_size = 10
tmp_embeding_size = 5
tmp_input_length = 8
tmp_batch_size = 2

tmp_model = tf.keras.models.Sequential()
tmp_model.add(tf.keras.layers.Embedding(
  input_dim=tmp_vocab_size,
  output_dim=tmp_embeding_size,
  input_length=tmp_input_length
))
# The model will take as input an integer matrix of size (batch, input_length).
# The largest integer (i.e. word index) in the input should be no larger than 9 (tmp_vocab_size).
# Now model.output_shape == (None, 10, 64), where None is the batch dimension.
tmp_input_array = np.random.randint(
  low=0,
  high=tmp_vocab_size,
  size=(tmp_batch_size, tmp_input_length)
)
tmp_model.compile('rmsprop', 'mse')
tmp_output_array = tmp_model.predict(tmp_input_array)

print('tmp_input_array shape:', tmp_input_array.shape)
print('tmp_input_array:')
print(tmp_input_array)
print()
print('tmp_output_array shape:', tmp_output_array.shape)
print('tmp_output_array:')
print(tmp_output_array)<jupyter_output>tmp_input_array shape: (2, 8)
tmp_input_array:
[[3 1 6 3 4 4 2 4]
 [6 9 6 3 7 9 5 1]]

tmp_output_array shape: (2, 8, 5)
tmp_output_array:
[[[-0.00902291 -0.04806514  0.01489567 -0.02260449 -0.01164066]
  [ 0.0126917  -0.02367562  0.00211167  0.02326207 -0.0419951 ]
  [-0.00801573  0.03197679 -0.00023288  0.01608399  0.03854393]
  [-0.00902291 -0.04806514  0.01489567 -0.02260449 -0.01164066]
  [-0.00050925 -0.02667323  0.03475242 -0.02922877 -0.03765874]
  [-0.00050925 -0.02667323  0.03475242 -0.02922877 -0.03765874]
  [ 0.02268226 -0.00235027 -0.03701643 -0.04210497  0.03283196]
  [-0.00050925 -0.02667323  0.03475242 -0.02922877 -0.03765874]]

 [[-0.00801573  0.03197679 -0.00023288  0.01608399  0.03854393]
  [ 0.01684548  0.01886963 -0.04056183  0.0430654   0.00069758]
  [-0.00801573  0.03197679 -0.00023288  0.01608399  0.03854393]
  [-0.00902291 -0.04806514  0.01489567 -0.02260449 -0.01164066]
  [ 0.04351832  0.04195533  0.04136122  0.02406046  0.0424086 ]
  [ 0.01684548  0.01886963 [...]<jupyter_text>Model 1<jupyter_code># Length of the vocabulary in chars.
vocab_size = VOCABULARY_SIZE

# The embedding dimension.
embedding_dim = 256

# Number of RNN units.
rnn_units = 1024
def build_model_1(vocab_size, embedding_dim, rnn_units, batch_size):
    model = tf.keras.models.Sequential()

    model.add(tf.keras.layers.Embedding(
        input_dim=vocab_size,
        output_dim=embedding_dim,
        batch_input_shape=[batch_size, None]
    ))

    model.add(tf.keras.layers.LSTM(
        units=rnn_units,
        return_sequences=True,
        stateful=True,
        recurrent_initializer=tf.keras.initializers.GlorotNormal()
    ))

    model.add(tf.keras.layers.Dense(vocab_size))
    
    return model
model_1 = build_model_1(vocab_size, embedding_dim, rnn_units, BATCH_SIZE)

model_1.summary()
tf.keras.utils.plot_model(
    model_1,
    show_shapes=True,
    show_layer_names=True,
    to_file='model_1.png'
)<jupyter_output><empty_output><jupyter_text>For each character the model looks up the embedding, runs the LSTM one time-step with the embedding as input, and applies the dense layer to generate logits predicting the log-likelihood of the next character:![Model architecture](https://www.tensorflow.org/tutorials/text/images/text_generation_training.png)Image source: [Text generation with an RNN](https://www.tensorflow.org/tutorials/text/text_generation) notebook.  Trying the model<jupyter_code>for input_example_batch, target_example_batch in dataset_train.take(1):
    example_batch_predictions = model_1(input_example_batch)
    print(example_batch_predictions.shape, "# (batch_size, sequence_length, vocab_size)")<jupyter_output>(128, 2000, 140) # (batch_size, sequence_length, vocab_size)<jupyter_text>To get actual predictions from the model we need to sample from the output distribution, to get actual character indices. This distribution is defined by the logits over the character vocabulary.<jupyter_code>print('Prediction for the 1st letter of the batch 1st sequense:')
print(example_batch_predictions[0, 0])
# Quick overview of how tf.random.categorical() works.

# logits is 2-D Tensor with shape [batch_size, num_classes].
# Each slice [i, :] represents the unnormalized log-probabilities for all classes.
# In the example below we say that the probability for class "0" is low but the
# probability for class "2" is much higher.
tmp_logits = [
  [-0.95, 0, 0.95],
];

# Let's generate 5 samples. Each sample is a class index. Class probabilities 
# are being taken into account (we expect to see more samples of class "2").
tmp_samples = tf.random.categorical(
    logits=tmp_logits,
    num_samples=5
)

print(tmp_samples)
sampled_indices = tf.random.categorical(
    logits=example_batch_predictions[0],
    num_samples=1
)

sampled_indices.shape
sampled_indices = tf.squeeze(
    input=sampled_indices,
    axis=-1
).numpy()

sampled_indices.shape
sampled_indices[:100]
print('Input:\n', repr(''.join(tokenizer.sequences_to_texts([input_example_batch[0].numpy()[:50]]))))
print()
print('Next char prediction:\n', repr(''.join(tokenizer.sequences_to_texts([sampled_indices[:50]]))))<jupyter_output>Input:
 "[ t i t l e ] \n \n j . p . ' s   b i g   d a d d y   b i s c u i t s \n \n [ i n g r e d i e n t s ] \n"

Next char prediction:
 'g ä w d ︎ ¼ * ¼ ì o ® 2 k ă ́ @ t v ư ‧ ö \x1a 2 y ”   a \x92 7 î 6 ¾ ~ k 7 € ì ⅞ w — ‧ | d a í = < ‚ g p'<jupyter_text>Trying the model with variable input<jupyter_code>for input_example_batch_custom, target_example_batch_custom in dataset_train.take(1):
    random_input = np.zeros(shape=(BATCH_SIZE, 10))
    example_batch_predictions_custom = model_1(random_input)
    print('Prediction shape: ', example_batch_predictions_custom.shape, "# (batch_size, sequence_length, vocab_size)\n")
    print('Custom length input: ')
    print(random_input)<jupyter_output>Prediction shape:  (128, 10, 140) # (batch_size, sequence_length, vocab_size)

Custom length input: 
[[0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 ...
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]
 [0. 0. 0. ... 0. 0. 0.]]<jupyter_text>Training the model  Attach an optimizer, and a loss function<jupyter_code># An objective function.
# The function is any callable with the signature scalar_loss = fn(y_true, y_pred).
def loss(labels, logits):
    return tf.keras.losses.sparse_categorical_crossentropy(
      y_true=labels,
      y_pred=logits,
      from_logits=True
    )

example_batch_loss = loss(target_example_batch, example_batch_predictions)

print("Prediction shape: ", example_batch_predictions.shape, " # (batch_size, sequence_length, vocab_size)")
print("scalar_loss:      ", example_batch_loss.numpy().mean())
adam_optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)

model_1.compile(
    optimizer=adam_optimizer,
    loss=loss
)<jupyter_output><empty_output><jupyter_text>Configuring checkpoints<jupyter_code>early_stopping_callback = tf.keras.callbacks.EarlyStopping(
    patience=5,
    monitor='loss',
    restore_best_weights=True,
    verbose=1
)
# Directory where the checkpoints will be saved.
checkpoint_dir = 'tmp/checkpoints'
os.makedirs(checkpoint_dir, exist_ok=True)

# Name of the checkpoint files
checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt_{epoch}')

checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_prefix,
    save_weights_only=True
)<jupyter_output><empty_output><jupyter_text>Execute the training<jupyter_code>history_1 = model_1.fit(
    x=dataset_train,
    epochs=50,
    steps_per_epoch=100,
    callbacks=[
        checkpoint_callback
        # early_stopping_callback
    ]
)
def render_training_history(training_history):
    loss = training_history.history['loss']
    plt.title('Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.plot(loss, label='Training set')
    plt.legend()
    plt.grid(linestyle='--', linewidth=1, alpha=0.5)
    plt.show()
render_training_history(history_1)<jupyter_output><empty_output><jupyter_text>Generating text  Restore the latest checkpointTo keep this prediction step simple, use a batch size of 1.Because of the way the RNN state is passed from timestep to timestep, the model only accepts a fixed batch size once built.To run the model with a different `batch_size`, we need to rebuild the model and restore the weights from the checkpoint.<jupyter_code>tf.train.latest_checkpoint(checkpoint_dir)
simplified_batch_size = 1

model_1_simplified = build_model_1(vocab_size, embedding_dim, rnn_units, simplified_batch_size)

model_1_simplified.load_weights(tf.train.latest_checkpoint(checkpoint_dir))

model_1_simplified.build(tf.TensorShape([simplified_batch_size, None]))
model_1_simplified.summary()
model_1_simplified.input_shape<jupyter_output><empty_output><jupyter_text>The prediction loopThe following code block generates the text:- It Starts by choosing a start string, initializing the RNN state and setting the number of characters to generate.- Get the prediction distribution of the next character using the start string and the RNN state.- Then, use a categorical distribution to calculate the index of the predicted character. Use this predicted character as our next input to the model.- The RNN state returned by the model is fed back into the model so that it now has more context, instead than only one character. After predicting the next character, the modified RNN states are again fed back into the model, which is how it learns as it gets more context from the previously predicted characters.![Prediction loop](https://www.tensorflow.org/tutorials/text/images/text_generation_sampling.png)Image source: [Text generation with an RNN](https://www.tensorflow.org/tutorials/text/text_generation) notebook.<jupyter_code># num_generate
# - number of characters to generate.
#
# temperature
# - Low temperatures results in more predictable text.
# - Higher temperatures results in more surprising text.
# - Experiment to find the best setting.
def generate_text(model, start_string, num_generate = 1000, temperature=1.0):
    # Evaluation step (generating text using the learned model)
    
    padded_start_string = STOP_WORD_TITLE + start_string.lower()

    # Converting our start string to numbers (vectorizing).
    input_indices = np.array(tokenizer.texts_to_sequences([padded_start_string]))

    # Empty string to store our results.
    text_generated = []

    # Here batch size == 1.
    model.reset_states()
    for char_index in range(num_generate):
        predictions = model(input_indices)
        # remove the batch dimension
        predictions = tf.squeeze(predictions, 0)

        # Using a categorical distribution to predict the character returned by the model.
        predictions = predictions / temperature
        predicted_id = tf.random.categorical(
            predictions,
            num_samples=1
        )[-1,0].numpy()

        # We pass the predicted character as the next input to the model
        # along with the previous hidden state.
        input_indices = tf.expand_dims([predicted_id], 0)
        
        next_character = tokenizer.sequences_to_texts(input_indices.numpy())[0]

        text_generated.append(next_character)

    return (padded_start_string + ''.join(text_generated))
generated_text = generate_text(
    model_1_simplified,
    start_string='O',
    num_generate = 100,
    temperature=1
)

print(generated_text)
generated_text = generate_text(
    model_1_simplified,
    start_string='Orange',
    num_generate = 100,
    temperature=1.0
)

print(generated_text)<jupyter_output>[title]

orangeº ⅞‱#äìﬂ`”&åîø@‧@“îă½¾>¿°™é“€¤~‱=%ư½¤å?ô‭%”€%—–‧◊¿€çó–@`™é_€¿⅛⅞¤€_\€–0_ëóô??_€<ư€–°⅓å€⅛_⅓_¼ëîº¾<jupyter_text>Save the model<jupyter_code>model_name = 'recipe_generation_rnn.h5'
model_1_simplified.save(model_name, save_format='h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13375094651338887, 'fraction_numerical': 0.043126591863426725, 'mean_word_length': 4.5618300153139355, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 81, 'https://': 12, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '657230', 'license': 'MIT', 'n_tokens_mistral': 12767, 'n_tokens_neox': 11730, 'n_words': 2633, 'path': 'experiments/recipe_generation_rnn/recipe_generation_rnn.ipynb', 'repo_name': 'ashishpatel26/machine-learning-experiments', 'size': '29054'}