<jupyter_start><jupyter_text><jupyter_code>import tensorflow as tf

from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
import numpy as np 
# !wget --no-check-certificate \
#     https://storage.googleapis.com/laurencemoroney-blog.appspot.com/irish-lyrics-eof.txt \
#     -O /tmp/irish-lyrics-eof.txt
tokenizer = Tokenizer()

data = open('./irish-lyrics-eof.txt').read()

corpus = data.lower().split("\n")

tokenizer.fit_on_texts(corpus)
total_words = len(tokenizer.word_index) + 1

print(tokenizer.word_index, '\n')
print(total_words)
input_sequences = []
for line in corpus:
    token_list = tokenizer.texts_to_sequences([line])[0]
    for i in range(1, len(token_list)):
        n_gram_sequence = token_list[:i+1]
        input_sequences.append(n_gram_sequence)

# pad sequences 
max_sequence_len = max([len(x) for x in input_sequences])
input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))

# create predictors and label
xs, labels = input_sequences[:,:-1],input_sequences[:,-1] # len(xs) = len(ys) 12038

ys = tf.keras.utils.to_categorical(labels, num_classes=total_words)
print(tokenizer.word_index['in'])
print(tokenizer.word_index['the'])
print(tokenizer.word_index['town'])
print(tokenizer.word_index['of'])
print(tokenizer.word_index['athy'])
print(tokenizer.word_index['one'])
print(tokenizer.word_index['jeremy'])
print(tokenizer.word_index['lanigan'])
print(xs[6], '\n')
print(ys[6])
print(xs[5], '\n')
print(ys[5])
model = Sequential()
model.add(Embedding(total_words, 100, input_length=max_sequence_len-1))
model.add(Bidirectional(LSTM(150)))
model.add(Dense(total_words, activation='softmax'))
adam = Adam(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
# earlystop = EarlyStopping(monitor='accuracy', min_delta=0.5, patience=70, verbose=1, mode='auto')
history = model.fit(xs, ys, epochs=100, verbose=1) # callbacks=earlystop
print(model.summary(), '\n')
print(model)
import matplotlib.pyplot as plt

def plot_graphs(history, string):
    plt.plot(history.history[string])
    plt.xlabel("Epochs")
    plt.ylabel(string)
    plt.title(string)
    plt.show()

plot_graphs(history, 'accuracy')
model.save('irish_lyrics.h5')
seed_text = "I've got a bad feeling about this"
next_words = 100
  
for _ in range(next_words):
    token_list = tokenizer.texts_to_sequences([seed_text])[0]
    token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')
    predicted = np.argmax(model.predict(token_list, verbose=0), axis=-1)
    output_word = ""
    for word, index in tokenizer.word_index.items():
        if index == predicted:
            output_word = word
            break
    seed_text += " " + output_word
print(seed_text)<jupyter_output>I've got a bad feeling about this noon the sash is torn away and you come again gone by tomorrow love you know me connaught round the night eyes of it was upon them easy sons away city night again began abusing free invitation vain now and heres her renown land down without corporal love and die in keep we bay again faces drawn dying under back back when summers gone by mooncoin gone fair in the magic they march with faces gone and gone by river and toome today gone all their way to dublin whack fol all raked and city but make him straight in
{'fraction_non_alphanumeric': 0.12080908839013577, 'fraction_numerical': 0.011637572734829594, 'mean_word_length': 5.516245487364621, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '557422', 'license': 'MIT', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1183, 'n_words': 294, 'path': 'Informatics/Deep Learning/TensorFlow - deeplearning.ai/3. NLP/Course_3_Week_4_Lesson_2_Notebook.ipynb', 'repo_name': 'MarcosSalib/Cocktail_MOOC', 'size': '3609'}