<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import regex as re

import nltk
from nltk.draw.dispersion import dispersion_plot
from nltk.corpus import stopwords
nltk.download('stopwords')
from nltk.probability import FreqDist
from textblob import TextBlob
from nltk.stem import WordNetLemmatizer
nltk.download('wordnet')
from nltk.corpus.reader.plaintext import PlaintextCorpusReader
nltk.download('averaged_perceptron_tagger')
from nltk.util import ngrams
from nltk.sentiment import SentimentIntensityAnalyzer
nltk.download('vader_lexicon')

from sklearn.preprocessing import LabelBinarizer

import tensorflow as tf
import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import LSTM
from keras.utils import np_utils

import urllib

import os

# Any results you write to the current directory are saved as output.
# from google.colab import drive
# drive.mount('/content/gdrive')
# print("The current directory is: ", os.getcwd())
# import os
# os.chdir("/content/gdrive/My Drive/<NAME>/<NAME>")
# print("The current directory is: ", os.getcwd())
txt = urllib.request.urlopen('http://www.textfiles.com/stories/3gables.txt').read().decode('utf8')
txt = txt.replace('\n', ' ')[322:]
txt[0:400]
# Let's do some cleaning
words = txt.split(' ')
words_s = [word.lower() for word in words if word.isalpha()]
len(words_s)
# |# creating characters, words and lists 
# characters = sorted(list(set(txt)))
# words = txt.split(' ')
# sentences = txt.split('.')
# # let's remove stop words from words
# stop_words = set(stopwords.words('english'))
# words_s = [i for i in words if not i in stop_words]
# words_s = [word.lower() for word in words_s if word.isalpha()]
# fdist = FreqDist(words_s) # checking most frequent words in whole document
# plt.figure(figsize= (12,2))
# plt.bar(pd.DataFrame(fdist.most_common()[0:20])[0], pd.DataFrame(fdist.most_common()[0:20])[1])
# plt.xticks(rotation=90)
# plt.show()
# # lemmatizing
# lemmatizer = WordNetLemmatizer()
# words_s_l = pd.Series(words_s).apply(lambda x: lemmatizer.lemmatize(x))
# words_s_l_p = pd.DataFrame(nltk.pos_tag(words_s_l), columns = ['words', 'pos'])
sentences = ' '.join(words_s)
# Let's look at overall sentiment of the book
scores = SentimentIntensityAnalyzer().polarity_scores(sentences)
print(scores)
# saving sentences as bigrams
def n_gram(x):
  tokens = [token for token in x.split(" ") if token != ""]
  output = list(ngrams(tokens, 3))
  return output
df = pd.DataFrame(pd.Series(sentences).apply(lambda x: n_gram(x))[0])

# make a dataframe with 2 words as independent and just next third word as target column
df = pd.concat([df[0] + ' ' + df[1], df[2]], axis = 1)
df.head()
df.shape
from sklearn.feature_extraction.text import TfidfVectorizer
tfid = TfidfVectorizer()
X = tfid.fit_transform(df[0]).toarray()
X.shape
encoder = LabelBinarizer()
y = encoder.fit_transform(df[2])
y.shape
# LSTMs accept input in the form of (number_of_sequences, length_of_sequence, number_of_features)
X = np.reshape(X, (X.shape[0], X.shape[1], 1))
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.LSTM(400, input_shape=(X.shape[1], X.shape[2]), return_sequences = True))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.LSTM(400, return_sequences = True))
model.add(tf.keras.layers.Dropout(0.2))
# model.add(tf.keras.layers.LSTM(500, return_sequences = True))
# model.add(tf.keras.layers.Dropout(0.2))
# model.add(tf.keras.layers.LSTM(500, return_sequences = True))
# model.add(tf.keras.layers.Dropout(0.2))
# model.add(tf.keras.layers.LSTM(2500, return_sequences = True))
# model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.LSTM(200))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.Dense(y.shape[1], activation='softmax'))
model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']
tf.logging.set_verbosity(tf.logging.INFO)

tpu_model = tf.contrib.tpu.keras_to_tpu_model(
    model,
    strategy=tf.contrib.tpu.TPUDistributionStrategy(
        tf.contrib.cluster_resolver.TPUClusterResolver(TPU_WORKER)))
tpu_model.summary()
tpu_model.fit(X, y, epochs = 40, batch_size = 100)
cpu_model = tpu_model.sync_to_cpu()
y_pred = cpu_model.predict(X[99:150])
(encoder.inverse_transform(y_pred)).tolist()
%%time
# Evaluate the model on valid set
score = cpu_model.evaluate(X, y, verbose = 0)

# Print test accuracy
print('\n', 'Valid accuracy:', score[1])
(encoder.inverse_transform(cpu_model.predict(X[1200:1201])))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14474807856532879, 'fraction_numerical': 0.018787361229718188, 'mean_word_length': 5.889705882352941, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '233651', 'license': 'MIT', 'n_tokens_mistral': 1758, 'n_tokens_neox': 1701, 'n_words': 369, 'path': 'deep_writing-tpu.ipynb', 'repo_name': 'Nik8x/Deep_writing_generating_text', 'size': '4698'}