<jupyter_start><jupyter_text>Text Prediction Using LSTM<jupyter_code>from google.colab import drive
drive.mount("/content/gdrive")
# Imports
import sys
import numpy as np
import string, os 
import pandas as pd
import random
import re
import seaborn as sns
import matplotlib.pyplot as plt
plt.style.use('dark_background')

import warnings
warnings.filterwarnings("ignore")
warnings.simplefilter(action='ignore', category=FutureWarning)
from keras.models import Sequential
from keras.layers import Dense, Dropout, LSTM
from keras.optimizers import RMSprop
text = '/content/gdrive/MyDrive/ti.txt'
raw_text = open(text, 'r', encoding='utf-8').read()
raw_text = raw_text.lower()
def clean_text(txt):
    punct = ['{','}','$','%','&','#','*','/',':',';','?','@','_','[',']']
    txt = "".join(v for v in txt if v not in punct).lower()
    txt = txt.encode("utf8").decode("ascii",'ignore')
    raw = ""
    for line in txt:
      stripped_line = line.rstrip('\n')
      raw += stripped_line
    new_text = ''.join(c for c in raw if not c.isdigit())
    return new_text
new_text = clean_text(raw_text)
new_text[0:8000]
chars = sorted(list(set(new_text)))
print(chars)
print(len(chars))
char_to_int = dict((c, i) for i, c in enumerate(chars))
char_to_int
int_to_char = dict((i, c) for i, c in enumerate(chars))
int_to_char
print('---Summary---')
n_chars = len(new_text)
n_vocab = len(chars)
print("corpus length by character ", n_chars)
print("Vocab length: ", n_vocab)
seq_length = 100 
step_length = 10

sentences = []    
next_chars = []   

for i in range(0, n_chars - seq_length, step_length): 
    sentences.append(new_text[i: i + seq_length])  #X
    next_chars.append(new_text[i + seq_length])  #y
sentences[0:10]
n_sentences = len(sentences)    # no of sentences 
n_sentences 
# creating a boolean array for each sentence, with no of characters as element

x = np.zeros((n_sentences, seq_length, n_vocab), dtype=np.bool)
y = np.zeros((n_sentences, n_vocab), dtype=np.bool)
# mapping chars to numbers
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        x[i, t, char_to_int[char]] = 1
    y[i, char_to_int[next_chars[i]]] = 1
x.shape,y.shape
# selecting the one with high prob like softmax
def sample(preds):
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds)
    exp_preds = np.exp(preds) #exp of log (x), isn't this same as x??
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1) 
    return np.argmax(probas)
## Double layer lstm

model = Sequential()
model.add(LSTM(128, input_shape=(seq_length, n_vocab), return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(128))
model.add(Dropout(0.2))
model.add(Dense(n_vocab, activation='softmax'))

optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
model.summary()
from keras.callbacks import ModelCheckpoint

filepath="saved_weights/saved_weights-{epoch:02d}-{loss:.4f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')
callbacks_list = [checkpoint]
# Fit the model

history = model.fit(x, y,batch_size=128,epochs=50,callbacks=callbacks_list)
model.save('double_layer_lstm_50epochs.h5')
from matplotlib import pyplot as plt
#plot the training and validation accuracy and loss at each epoch
loss = history.history['loss']
epochs = range(1, len(loss) + 1)
plt.plot(epochs, loss, 'y', label='Training loss')
plt.title('Training loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()
def fin_pred(preds):
  preds = np.asarray(preds).astype('float64')
  preds = np.log(preds)
  exp_preds = np.exp(preds)
  preds = exp_preds / np.sum(exp_preds)
  probas = np.random.multinomial(1, preds, 1) 
  return np.argmax(probas)
def output(char_len):
  start_index = random.randint(0, n_chars - seq_length - 1)
  generated_text =''
  sentence= raw_text[start_index: start_index + seq_length]
  generated_text += sentence
  print(" Seed for prediction: "+sentence + "--->")
  for i in range(char_len):
    x_pred = np.zeros((1, seq_length, n_vocab))
    for t, char in enumerate(sentence):
      red_list = ["\n","'",';','"']
      if char not in red_list:
        x_pred[0, t, char_to_int[char]] = 1.
    preds = model.predict(x_pred, verbose=0)[0]
    next_index = fin_pred(preds)
    next_char = int_to_char[next_index]

    generated_text += next_char
    sentence = sentence[1:] + next_char
    sys.stdout.write(next_char)
    sys.stdout.flush()
print()
output(400)
gen = output(10000)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14767009407131917, 'fraction_numerical': 0.015313935681470138, 'mean_word_length': 4.945383615084525, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '658223', 'license': 'MIT', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1686, 'n_words': 392, 'path': 'Text Prediction/Text_Prediction_LSTM.ipynb', 'repo_name': 'rthirumurugan2000/Machine-Learning-Projects', 'size': '4571'}