<jupyter_start><jupyter_text>Machine TranslationEnglish to french<jupyter_code>import collections

import numpy as np

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Model
from tensorflow.keras.layers import GRU, Input, Dense, TimeDistributed, Activation, RepeatVector, Bidirectional
from tensorflow.keras.layers import Embedding
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import sparse_categorical_crossentropy<jupyter_output><empty_output><jupyter_text>Verify access to the GPU<jupyter_code>from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())<jupyter_output>[name: "/device:CPU:0"
device_type: "CPU"
memory_limit: 268435456
locality {
}
incarnation: 14719032101380401830
, name: "/device:GPU:0"
device_type: "GPU"
memory_limit: 2254123828
locality {
  bus_id: 1
  links {
  }
}
incarnation: 15774481516893467955
physical_device_desc: "device: 0, name: NVIDIA GeForce GTX 1650 Ti, pci bus id: 0000:01:00.0, compute capability: 7.5"
]<jupyter_text>Dataset[WMT](http://www.statmt.org/)<jupyter_code>import os
def load_data(path):
    """
    Load dataset
    """
    input_file = os.path.join(path)
    with open(input_file, "r") as f:
        data = f.read()

    return data.split('\n')
# Load English data
english_sentences = load_data('small_vocab_en')
# Load French data
french_sentences = load_data('small_vocab_fr')

print('Dataset Loaded')<jupyter_output>Dataset Loaded<jupyter_text>Files<jupyter_code>for sample_i in range(2):
    print('small_vocab_en Line {}:  {}'.format(sample_i + 1, english_sentences[sample_i]))
    print('small_vocab_fr Line {}:  {}'.format(sample_i + 1, french_sentences[sample_i]))<jupyter_output>small_vocab_en Line 1:  new jersey is sometimes quiet during autumn , and it is snowy in april .
small_vocab_fr Line 1:  new jersey est parfois calme pendant l' automne , et il est neigeux en avril .
small_vocab_en Line 2:  the united states is usually chilly during july , and it is usually freezing in november .
small_vocab_fr Line 2:  les Ã©tats-unis est gÃ©nÃ©ralement froid en juillet , et il gÃ¨le habituellement en novembre .<jupyter_text>Vocabulary<jupyter_code>english_words_counter = collections.Counter([word for sentence in english_sentences for word in sentence.split()])
french_words_counter = collections.Counter([word for sentence in french_sentences for word in sentence.split()])

print('{} English words.'.format(len([word for sentence in english_sentences for word in sentence.split()])))
print('{} unique English words.'.format(len(english_words_counter)))
print('10 Most common words in the English dataset:')
print('"' + '" "'.join(list(zip(*english_words_counter.most_common(10)))[0]) + '"')
print()
print('{} French words.'.format(len([word for sentence in french_sentences for word in sentence.split()])))
print('{} unique French words.'.format(len(french_words_counter)))
print('10 Most common words in the French dataset:')
print('"' + '" "'.join(list(zip(*french_words_counter.most_common(10)))[0]) + '"')<jupyter_output>1823250 English words.
227 unique English words.
10 Most common words in the English dataset:
"is" "," "." "in" "it" "during" "the" "but" "and" "sometimes"

1961295 French words.
355 unique French words.
10 Most common words in the French dataset:
"est" "." "," "en" "il" "les" "mais" "et" "la" "parfois"<jupyter_text>Preprocess Tokenize (IMPLEMENTATION)<jupyter_code>def tokenize(x):
    x_tk = Tokenizer(char_level = False)
    x_tk.fit_on_texts(x)
    return x_tk.texts_to_sequences(x), x_tk

# Tokenize Example output
text_sentences = [
    'The quick brown fox jumps over the lazy dog .',
    'By Jove , my quick study of lexicography won a prize .',
    'This is a short sentence .']
text_tokenized, text_tokenizer = tokenize(text_sentences)
print(text_tokenizer.word_index)
print()
for sample_i, (sent, token_sent) in enumerate(zip(text_sentences, text_tokenized)):
    print('Sequence {} in x'.format(sample_i + 1))
    print('  Input:  {}'.format(sent))
    print('  Output: {}'.format(token_sent))<jupyter_output>{'the': 1, 'quick': 2, 'a': 3, 'brown': 4, 'fox': 5, 'jumps': 6, 'over': 7, 'lazy': 8, 'dog': 9, 'by': 10, 'jove': 11, 'my': 12, 'study': 13, 'of': 14, 'lexicography': 15, 'won': 16, 'prize': 17, 'this': 18, 'is': 19, 'short': 20, 'sentence': 21}

Sequence 1 in x
  Input:  The quick brown fox jumps over the lazy dog .
  Output: [1, 2, 4, 5, 6, 7, 1, 8, 9]
Sequence 2 in x
  Input:  By Jove , my quick study of lexicography won a prize .
  Output: [10, 11, 12, 2, 13, 14, 15, 16, 3, 17]
Sequence 3 in x
  Input:  This is a short sentence .
  Output: [18, 19, 3, 20, 21]<jupyter_text>Padding (IMPLEMENTATION)<jupyter_code>def pad(x, length=None):
    if length is None:
        length = max([len(sentence) for sentence in x])
    return pad_sequences(x, maxlen = length, padding = 'post')


# Pad Tokenized output
test_pad = pad(text_tokenized)
for sample_i, (token_sent, pad_sent) in enumerate(zip(text_tokenized, test_pad)):
    print('Sequence {} in x'.format(sample_i + 1))
    print('  Input:  {}'.format(np.array(token_sent)))
    print('  Output: {}'.format(pad_sent))<jupyter_output>Sequence 1 in x
  Input:  [1 2 4 5 6 7 1 8 9]
  Output: [1 2 4 5 6 7 1 8 9 0]
Sequence 2 in x
  Input:  [10 11 12  2 13 14 15 16  3 17]
  Output: [10 11 12  2 13 14 15 16  3 17]
Sequence 3 in x
  Input:  [18 19  3 20 21]
  Output: [18 19  3 20 21  0  0  0  0  0]<jupyter_text>Preprocess Pipeline<jupyter_code>def preprocess(x, y):
    preprocess_x, x_tk = tokenize(x)
    preprocess_y, y_tk = tokenize(y)

    preprocess_x = pad(preprocess_x)
    preprocess_y = pad(preprocess_y)

    # Keras's sparse_categorical_crossentropy function requires the labels to be in 3 dimensions
    preprocess_y = preprocess_y.reshape(*preprocess_y.shape, 1)

    return preprocess_x, preprocess_y, x_tk, y_tk

preproc_english_sentences, preproc_french_sentences, english_tokenizer, french_tokenizer =\
    preprocess(english_sentences, french_sentences)
    
max_english_sequence_length = preproc_english_sentences.shape[1]
max_french_sequence_length = preproc_french_sentences.shape[1]
english_vocab_size = len(english_tokenizer.word_index)
french_vocab_size = len(french_tokenizer.word_index)

print('Data Preprocessed')
print("Max English sentence length:", max_english_sequence_length)
print("Max French sentence length:", max_french_sequence_length)
print("English vocabulary size:", english_vocab_size)
print("French vocabulary size:", french_vocab_size)
preproc_english_sentences
english_tokenizer
preproc_french_sentences
french_tokenizer<jupyter_output><empty_output><jupyter_text>Models- Model 1 is a simple RNN- Model 2 is a RNN with Embedding- Model 3 is a Bidirectional RNN- Model 4 is an optional Encoder-Decoder RNN Ids Back to TextThe neural network will be translating the input to words ids, which isn't the final form we want.  We want the French translation.  The function `logits_to_text` will bridge the gab between the logits from the neural network to the French translation.  You'll be using this function to better understand the output of the neural network.<jupyter_code>def logits_to_text(logits, tokenizer):
    index_to_words = {id: word for word, id in tokenizer.word_index.items()}
    index_to_words[0] = '<PAD>'

    return ' '.join([index_to_words[prediction] for prediction in np.argmax(logits, 1)])

print('`logits_to_text` function loaded.')<jupyter_output>`logits_to_text` function loaded.<jupyter_text>Model 1: RNN (IMPLEMENTATION)A basic RNN model is a good baseline for sequence data.  In this model, you'll build a RNN that translates English to French.<jupyter_code>def simple_model(input_shape, output_sequence_length, english_vocab_size, french_vocab_size):
    """
    Build and train a basic RNN on x and y
    :param input_shape: Tuple of input shape
    :param output_sequence_length: Length of output sequence
    :param english_vocab_size: Number of unique English words in the dataset
    :param french_vocab_size: Number of unique French words in the dataset
    :return: Keras model built, but not trained
    """
    learning_rate = 1e-3
    input_seq = Input(input_shape[1:])
    rnn = GRU(64, return_sequences = True)(input_seq)
    logits = TimeDistributed(Dense(french_vocab_size))(rnn)
    model = Model(input_seq, Activation('softmax')(logits))
    model.compile(loss = sparse_categorical_crossentropy, 
                 optimizer = Adam(learning_rate), 
                 metrics = ['accuracy'])
    
    return model

# Reshaping the input to work with a basic RNN
tmp_x = pad(preproc_english_sentences, max_french_sequence_length)
tmp_x = tmp_x.reshape((-1, preproc_french_sentences.shape[-2], 1))
tmp_x.shape
# Train the neural network
simple_rnn_model = simple_model(
    tmp_x.shape,
    max_french_sequence_length,
    english_vocab_size,
    french_vocab_size)
simple_rnn_model.summary()
simple_rnn_model.fit(tmp_x, preproc_french_sentences, batch_size=1024, epochs=10, validation_split=0.2)
# Print prediction(s)
print(logits_to_text(simple_rnn_model.predict(tmp_x[:1])[0], french_tokenizer))<jupyter_output>paris jersey est parfois parfois en l' et il est est en en <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD> <PAD><jupyter_text>Model 2: Embedding (IMPLEMENTATION)<jupyter_code>from tensorflow.keras.models import Sequential
def embed_model(input_shape, output_sequence_length, english_vocab_size, french_vocab_size):
    """
    Build and train a RNN model using word embedding on x and y
    :param input_shape: Tuple of input shape
    :param output_sequence_length: Length of output sequence
    :param english_vocab_size: Number of unique English words in the dataset
    :param french_vocab_size: Number of unique French words in the dataset
    :return: Keras model built, but not trained
    """
    learning_rate = 1e-3
    rnn = GRU(64, return_sequences=True, activation="tanh")
    
    embedding = Embedding(french_vocab_size, 64, input_length=input_shape[1]) 
    logits = TimeDistributed(Dense(french_vocab_size, activation="softmax"))
    #TimeDistributed
    #This wrapper allows to apply a layer to every temporal slice of an input.
    #Every input should be at least 3D, and the dimension of index one of the first
    #input will be considered to be the temporal dimension.
    
    model = Sequential()
    #em can only be used in first layer --> Keras Documentation
    model.add(embedding)
    model.add(rnn)
    model.add(logits)
    model.compile(loss=sparse_categorical_crossentropy,
                  optimizer=Adam(learning_rate),
                  metrics=['accuracy'])
    
    return model


# TODO: Reshape the input
tmp_x = pad(preproc_english_sentences, max_french_sequence_length)
tmp_x = tmp_x.reshape((-1, preproc_french_sentences.shape[-2]))

# TODO: Train the neural network

embeded_model = embed_model(
    tmp_x.shape,
    max_french_sequence_length,
    english_vocab_size,
    french_vocab_size)

embeded_model.fit(tmp_x, preproc_french_sentences, batch_size=1024, epochs=10, validation_split=0.2)


# TODO: Print prediction(s)
print(logits_to_text(embeded_model.predict(tmp_x[:1])[0], french_tokenizer))<jupyter_output>Epoch 1/10
108/108 [==============================] - 5s 28ms/step - loss: 3.7081 - accuracy: 0.4045 - val_loss: nan - val_accuracy: 0.4093
Epoch 2/10
108/108 [==============================] - 3s 25ms/step - loss: 2.6648 - accuracy: 0.4480 - val_loss: nan - val_accuracy: 0.5085
Epoch 3/10
108/108 [==============================] - 3s 24ms/step - loss: 2.0031 - accuracy: 0.5569 - val_loss: nan - val_accuracy: 0.5972
Epoch 4/10
108/108 [==============================] - 3s 24ms/step - loss: 1.5022 - accuracy: 0.6342 - val_loss: nan - val_accuracy: 0.6802
Epoch 5/10
108/108 [==============================] - 3s 24ms/step - loss: 1.1564 - accuracy: 0.7163 - val_loss: nan - val_accuracy: 0.7464
Epoch 6/10
108/108 [==============================] - 3s 25ms/step - loss: 0.9477 - accuracy: 0.7606 - val_loss: nan - val_accuracy: 0.7740
Epoch 7/10
108/108 [==============================] - 3s 25ms/step - loss: 0.8094 - accuracy: 0.7857 - val_loss: nan - val_accuracy: 0.7981
Epoch 8/10
108/108 [[...]<jupyter_text>Model 3: Bidirectional RNNs (IMPLEMENTATION)<jupyter_code>def bd_model(input_shape, output_sequence_length, english_vocab_size, french_vocab_size):
    """
    Build and train a bidirectional RNN model on x and y
    :param input_shape: Tuple of input shape
    :param output_sequence_length: Length of output sequence
    :param english_vocab_size: Number of unique English words in the dataset
    :param french_vocab_size: Number of unique French words in the dataset
    :return: Keras model built, but not trained
    """

    learning_rate = 1e-3
    model = Sequential()
    model.add(Bidirectional(GRU(128, return_sequences = True, dropout = 0.1), 
                           input_shape = input_shape[1:]))
    model.add(TimeDistributed(Dense(french_vocab_size, activation = 'softmax')))
    model.compile(loss = sparse_categorical_crossentropy, 
                 optimizer = Adam(learning_rate), 
                 metrics = ['accuracy'])
    return model



tmp_x = pad(preproc_english_sentences, preproc_french_sentences.shape[1])
tmp_x = tmp_x.reshape((-1, preproc_french_sentences.shape[-2], 1))

bidi_model = bd_model(
    tmp_x.shape,
    preproc_french_sentences.shape[1],
    len(english_tokenizer.word_index)+1,
    len(french_tokenizer.word_index)+1)


bidi_model.fit(tmp_x, preproc_french_sentences, batch_size=1024, epochs=20, validation_split=0.2)

# Print prediction(s)
print(logits_to_text(bidi_model.predict(tmp_x[:1])[0], french_tokenizer))<jupyter_output>Epoch 1/20
108/108 [==============================] - 9s 45ms/step - loss: 2.6458 - accuracy: 0.4964 - val_loss: 1.8190 - val_accuracy: 0.5621
Epoch 2/20
108/108 [==============================] - 4s 39ms/step - loss: 1.6989 - accuracy: 0.5768 - val_loss: 1.5023 - val_accuracy: 0.6061
Epoch 3/20
108/108 [==============================] - 4s 39ms/step - loss: 1.4799 - accuracy: 0.6072 - val_loss: 1.3891 - val_accuracy: 0.6167
Epoch 4/20
108/108 [==============================] - 4s 39ms/step - loss: 1.3672 - accuracy: 0.6242 - val_loss: 1.3297 - val_accuracy: 0.6200
Epoch 5/20
108/108 [==============================] - 4s 39ms/step - loss: 1.2887 - accuracy: 0.6383 - val_loss: 1.2956 - val_accuracy: 0.6184
Epoch 6/20
108/108 [==============================] - 4s 40ms/step - loss: 1.2310 - accuracy: 0.6493 - val_loss: 1.2897 - val_accuracy: 0.6169
Epoch 7/20
108/108 [==============================] - 4s 40ms/step - loss: 1.1847 - accuracy: 0.6576 - val_loss: 1.3100 - val_accuracy: 0.6119[...]<jupyter_text>Model 4: Encoder-Decoder<jupyter_code>def encdec_model(input_shape, output_sequence_length, english_vocab_size, french_vocab_size):
    """
    Build and train an encoder-decoder model on x and y
    :param input_shape: Tuple of input shape
    :param output_sequence_length: Length of output sequence
    :param english_vocab_size: Number of unique English words in the dataset
    :param french_vocab_size: Number of unique French words in the dataset
    :return: Keras model built, but not trained
    """

    learning_rate = 1e-3
    model = Sequential()
    model.add(GRU(128, input_shape = input_shape[1:], return_sequences = False))
    model.add(RepeatVector(output_sequence_length))
    model.add(GRU(128, return_sequences = True))
    model.add(TimeDistributed(Dense(french_vocab_size, activation = 'softmax')))
    
    model.compile(loss = sparse_categorical_crossentropy, 
                 optimizer = Adam(learning_rate), 
                 metrics = ['accuracy'])
    return model


tmp_x = pad(preproc_english_sentences)
tmp_x = tmp_x.reshape((-1, preproc_english_sentences.shape[1], 1))

encodeco_model = encdec_model(
    tmp_x.shape,
    preproc_french_sentences.shape[1],
    len(english_tokenizer.word_index)+1,
    len(french_tokenizer.word_index)+1)

encodeco_model.fit(tmp_x, preproc_french_sentences, batch_size=1024, epochs=20, validation_split=0.2)

print(logits_to_text(encodeco_model.predict(tmp_x[:1])[0], french_tokenizer))<jupyter_output>Epoch 1/20
108/108 [==============================] - 8s 43ms/step - loss: 2.9813 - accuracy: 0.4430 - val_loss: 2.4524 - val_accuracy: 0.4955
Epoch 2/20
108/108 [==============================] - 4s 37ms/step - loss: 2.3107 - accuracy: 0.5001 - val_loss: 2.2085 - val_accuracy: 0.5067
Epoch 3/20
108/108 [==============================] - 4s 37ms/step - loss: 2.0856 - accuracy: 0.5163 - val_loss: 1.9217 - val_accuracy: 0.5253
Epoch 4/20
108/108 [==============================] - 4s 37ms/step - loss: 1.8331 - accuracy: 0.5477 - val_loss: 1.7507 - val_accuracy: 0.5597
Epoch 5/20
108/108 [==============================] - 4s 37ms/step - loss: 1.7005 - accuracy: 0.5677 - val_loss: 1.6518 - val_accuracy: 0.5773
Epoch 6/20
108/108 [==============================] - 4s 37ms/step - loss: 1.6159 - accuracy: 0.5773 - val_loss: 1.5780 - val_accuracy: 0.5849
Epoch 7/20
108/108 [==============================] - 4s 37ms/step - loss: 1.5466 - accuracy: 0.5872 - val_loss: 1.5179 - val_accuracy: 0.5885[...]<jupyter_text>Model 5: Custom (IMPLEMENTATION)<jupyter_code>def model_final(input_shape, output_sequence_length, english_vocab_size, french_vocab_size):
    """
    Build and train a model that incorporates embedding, encoder-decoder, and bidirectional RNN on x and y
    :param input_shape: Tuple of input shape
    :param output_sequence_length: Length of output sequence
    :param english_vocab_size: Number of unique English words in the dataset
    :param french_vocab_size: Number of unique French words in the dataset
    :return: Keras model built, but not trained
    """
    model = Sequential()
    model.add(Embedding(input_dim=english_vocab_size,output_dim=128,input_length=input_shape[1]))
    model.add(Bidirectional(GRU(256,return_sequences=False)))
    model.add(RepeatVector(output_sequence_length))
    model.add(Bidirectional(GRU(256,return_sequences=True)))
    model.add(TimeDistributed(Dense(french_vocab_size,activation='softmax')))
    learning_rate = 0.005
    
    model.compile(loss = sparse_categorical_crossentropy, 
                 optimizer = Adam(learning_rate), 
                 metrics = ['accuracy'])
    
    return model



print('Final Model Loaded')
# TODO: Train the final model<jupyter_output>Final Model Loaded<jupyter_text>Prediction (IMPLEMENTATION)<jupyter_code>len(english_tokenizer.word_index)+1, len(french_tokenizer.word_index)+1
from tensorflow.keras.models import Sequential

"""
Gets predictions using the final model
:param x: Preprocessed English data
:param y: Preprocessed French data
:param x_tk: English tokenizer
:param y_tk: French tokenizer
"""

tmp_X = pad(preproc_english_sentences)
model = model_final(tmp_X.shape,
                    preproc_french_sentences.shape[1],
                    len(english_tokenizer.word_index)+1,
                    len(french_tokenizer.word_index)+1)

model.fit(tmp_X, preproc_french_sentences, batch_size = 1024, epochs = 17, validation_split = 0.2)

y_id_to_word = {value: key for key, value in french_tokenizer.word_index.items()}
y_id_to_word[0] = '<PAD>'
y_id_to_word
sentence = 'he saw a old yellow truck'
sentence = [english_tokenizer.word_index[word] for word in sentence.split()]
sentence
sentence = pad_sequences([sentence], maxlen=preproc_english_sentences.shape[-1], padding='post')
sentence
sentences = np.array([sentence[0], preproc_english_sentences[0]])
sentence
predictions = model.predict(sentences, len(sentences))
print('Sample 1:')
print(' '.join([y_id_to_word[np.argmax(x)] for x in predictions[0]]))
print('Il a vu un vieux camion jaune')
print('Sample 2:')
print(' '.join([y_id_to_word[np.argmax(x)] for x in predictions[1]]))
print(' '.join([y_id_to_word[np.max(x)] for x in preproc_french_sentences[0]]))
import pickle

# serialize model to JSON
model_json = model.to_json()
with open("translator_model.json", "w") as json_file:
    json_file.write(model_json)
    
# saving model
model.save('translator_model.h5')
print("Model saved")

# saving tokenizer
with open('english_tokenizer.pickle', 'wb') as handle:
    pickle.dump(english_tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)

# saving tokenizer
with open('french_tokenizer.pickle', 'wb') as handle:
    pickle.dump(french_tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
print('Tokenizer saved')<jupyter_output>Model saved
Tokenizer saved
{'fraction_non_alphanumeric': 0.13782735208535402, 'fraction_numerical': 0.05140640155189137, 'mean_word_length': 4.794043270581624, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '80590', 'license': 'MIT', 'n_tokens_mistral': 7928, 'n_tokens_neox': 7066, 'n_words': 1811, 'path': 'Training/.ipynb_checkpoints/machine_translation-checkpoint.ipynb', 'repo_name': 'choudhury722k/English-to-French-translator', 'size': '20620'}