<jupyter_start><jupyter_text>1. IntroductionNotebook for generating lyrics using LSTM network. The dataset contains all the songs recorded by <NAME>. Stages:1. EDA    - Summary statistics on dataset: distribution of no. of characters, words, sentences in collection    - Histograms & wordclouds2. Preprocessing    - Create corpus of all words from lyrics    - Cleaning: remove special characters, convert to lowercase    - Create mapping of unique chars to indices    - Create features and targets (categorical)3. Model    - Train LSTM model, one character at a time    - Visualize learning and loss4. Generation    - Generate lyrics from seed phrase, one character at a time, using model predictions<jupyter_code># Imports
# Core
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import string, os, sys, warnings, random, io
warnings.filterwarnings("ignore")

# NLP
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import LSTM, Dense, Dropout
import nltk
import re
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from PIL import Image
# TPU config (for LSTM)
tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()
tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)<jupyter_output><empty_output><jupyter_text>2. EDA<jupyter_code># Load song lyrics dataset
df_songs = pd.read_csv("../input/bob-dylan-songs/clear.csv")
df_songs.head()
df_songs.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 345 entries, 0 to 344
Data columns (total 4 columns):
 #   Column        Non-Null Count  Dtype 
---  ------        --------------  ----- 
 0   release_year  345 non-null    int64 
 1   album         345 non-null    object
 2   title         345 non-null    object
 3   lyrics        345 non-null    object
dtypes: int64(1), object(3)
memory usage: 10.9+ KB<jupyter_text>2.1 Summary stats<jupyter_code># Get numbers of characters,words and sentences in each song
df_songs["n_chars"] = df_songs["lyrics"].apply(len)
df_songs["n_words"]=df_songs.apply(lambda row: nltk.word_tokenize(row["lyrics"]), axis=1).apply(len)
df_songs["n_lines"] = df_songs["lyrics"].str.split('\n').apply(len)
df_songs.describe()
# Plot distribution of chars, words, sentences in lyrics
fig, axs = plt.subplots(3, 1, figsize=(15, 10))
fig.suptitle('Distribution of characters, words and lines')
sns.histplot(data=df_songs, x='n_chars', ax=axs[0])
sns.histplot(data=df_songs, x='n_words', ax=axs[1])
sns.histplot(data=df_songs, x='n_lines', ax=axs[2])<jupyter_output><empty_output><jupyter_text>2.2 Wordcloud<jupyter_code># Generate a wordcloud
stopwords = set(STOPWORDS)
wc = WordCloud(stopwords=stopwords, background_color="#007399",colormap="cividis", max_words=100)
wc.generate(" ".join(df_songs.lyrics))
plt.figure(figsize=(12,12))
plt.imshow(wc, interpolation="bilinear")
plt.show()
# Sample lyrics
song_idx = 100
print(df_songs.loc[song_idx, :])
df_songs.loc[song_idx, 'lyrics'].split('\n')[:30]<jupyter_output>release_year                                                 1967
album                                         <NAME>
title                  The Ballad of Frankie Lee and Judas Priest
lyrics          Well, Frankie Lee and Judas Priest\n \n They w...
n_chars                                                      3160
n_words                                                       682
n_lines                                                       195
Name: 100, dtype: object<jupyter_text>3. Preprocessing<jupyter_code>df_songs.shape<jupyter_output><empty_output><jupyter_text>3.1 Create corpus<jupyter_code># Create corpus from lyrics
corpus = ''
for text in df_songs.lyrics:
    corpus += text
    
corpus = corpus.lower()
print("Number of unique characters:", len(set(corpus)))<jupyter_output>Number of unique characters: 65<jupyter_text>3.2 Cleaning<jupyter_code>print(sorted(set(corpus)))
special_chars = ['\x0b', '"', '(', ')', '`', '¥', '©', 'é', 'ñ', 'ó', 'ü', '—', '“', '”', '…', '\u2028']
for symbol in special_chars:
    corpus = corpus.replace(symbol,"")
# corpus = re.sub("[^A-Za-z0-9'.,?!\n\w]","",corpus)
print(sorted(set(corpus)))
# Sample section
corpus[:1000]<jupyter_output><empty_output><jupyter_text>3.3 Create inputs from mapping<jupyter_code># Map characters in corpus to indices
symb = sorted(list(set(corpus)))

len_corpus = len(corpus) 
len_symb = len(symb)

mapping = dict((c, i) for i, c in enumerate(symb))
reverse_mapping = dict((i, c) for i, c in enumerate(symb))

print("Total number of characters:", len_corpus)
print("Number of unique characters:", len_symb)
#Splitting the corpus in equal length of strings and output target
length = 50
features = []
targets = []
for i in range(0, len_corpus - length, 1):
    feature = corpus[i:i + length]
    target = corpus[i + length]
    features.append([mapping[j] for j in feature])
    targets.append(mapping[target])
    
# len_datapoints = len(targets)
# print("Total number of sequences in the Corpus:", len_datapoints)
# reshape X and normalize
X = (np.reshape(features, (len(targets), length, 1)))/ float(len_symb)

# one hot encode the output variable
y = to_categorical(targets)<jupyter_output><empty_output><jupyter_text>4. Model  4.1 Training<jupyter_code>#Initialising the Model
with tpu_strategy.scope():
    model = Sequential()
    model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))
    model.add(Dense(y.shape[1], activation='softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam')
model.summary()
%%time
#Training the Model
history = model.fit(X, y, batch_size=256*tpu_strategy.num_replicas_in_sync, epochs=50)<jupyter_output>Epoch 1/50
268/268 [==============================] - 9s 17ms/step - loss: 3.0756
Epoch 2/50
268/268 [==============================] - 4s 14ms/step - loss: 2.8562
Epoch 3/50
268/268 [==============================] - 4s 14ms/step - loss: 2.7715
Epoch 4/50
268/268 [==============================] - 4s 14ms/step - loss: 2.7011
Epoch 5/50
268/268 [==============================] - 4s 14ms/step - loss: 2.6631
Epoch 6/50
268/268 [==============================] - 4s 14ms/step - loss: 2.6197
Epoch 7/50
268/268 [==============================] - 4s 14ms/step - loss: 2.5924
Epoch 8/50
268/268 [==============================] - 4s 14ms/step - loss: 2.5620
Epoch 9/50
268/268 [==============================] - 4s 15ms/step - loss: 2.5384
Epoch 10/50
268/268 [==============================] - 4s 14ms/step - loss: 2.5131
Epoch 11/50
268/268 [==============================] - 4s 14ms/step - loss: 2.4917
Epoch 12/50
268/268 [==============================] - 4s 14ms/step - loss: 2.4776
Epoch 13/50
2[...]<jupyter_text>model.save("generator_v2.h5")  4.2 Evaluation<jupyter_code># Evaluation
history_df = pd.DataFrame(history.history)
#Plotting the learning curve

fig = plt.figure(figsize=(12,6))
fig.suptitle("Learning plot for model Loss")
pl = sns.lineplot(data=history_df["loss"])
pl.set(ylabel ="Training Loss")
pl.set(xlabel ="Epochs")<jupyter_output><empty_output><jupyter_text>5. Generation<jupyter_code># The function to generate text from model
def generate(seed,char_count, temperature=1.0):
    generated= ""
    seed = seed 
    seed_idx=[mapping[char] for char in seed]
    generated += seed 
    # Generating new text of given length
    for i in range(char_count):
        seed_idx=[mapping[char] for char in seed]
        x_pred = np.reshape(seed_idx, (1, len(seed_idx), 1))/len_symb
        prediction = model.predict(x_pred, verbose=0)[0]  
        # Getting the index of the next most probable index
        prediction = np.asarray(prediction).astype('float64')
        prediction = np.log(prediction) / temperature
#         exp_preds = np.exp(prediction)
#         prediction = exp_preds / np.sum(exp_preds)
#         probas = np.random.multinomial(1, prediction, 1)
        index = np.argmax(prediction)
        next_char = reverse_mapping[index]  
        # Generating new text
        generated += next_char
        seed = seed[1:] + next_char
       
    return generated
song = generate("the answer my friend, is blowi", 50, 2.0)
song.split('\n')
song = generate("the times they ar", 200)
song.split('\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16640671865626874, 'fraction_numerical': 0.04379124175164967, 'mean_word_length': 3.9826658696951585, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '8598', 'license': 'Apache-2.0', 'n_tokens_mistral': 3051, 'n_tokens_neox': 2742, 'n_words': 702, 'path': 'lyrics-generation-with-lstm.ipynb', 'repo_name': 'abhinav-neil/lyrics-generator-lstm', 'size': '8351'}