<jupyter_start><jupyter_text>Copyright 2019 <NAME>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License athttps://www.apache.org/licenses/LICENSE-2.0Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.     Run in Google ColabView source on GitHub  Pre-processing  Download the libraries and datasetCheck the current directory and upload the text file:<jupyter_code>import os
fpath = os.getcwd(); fpath
# use for file uploading into Google Colab Files folder
from google.colab import files

uploaded = files.upload()

for fn in uploaded.keys():
    print('User uploaded file "{name}" with length {length} bytes'.format(
      name=fn, length=len(uploaded[fn])))
  
# Click the Files tab on the left - the uploaded file(s) will be there
book = open(fpath + '/Plutarch.txt', 'rb').read().decode(encoding='utf-8')
words = [w for w in book.split(' ') if w.strip() != '' or w == '\n']
print("The text is {} words long".format(len(words)))<jupyter_output><empty_output><jupyter_text>Explore the textTokenize, convert to lower case, remove stop words and some punctuation, then save the new text for future reference:<jupyter_code># Check the English stop words in case need to enhance the list
import nltk
nltk.download('stopwords') #need if using Google Colab
from nltk.corpus import stopwords
print (stopwords.words('english'))
import string
from nltk.tokenize import word_tokenize
nltk.download('punkt') #need if using Google Colab
import matplotlib.pyplot as plt
%matplotlib inline

with open(fpath + '/Plutarch.txt') as f, open(fpath + '/Plutarch_tokens.txt', 'w') as out_f:
    text = f.read()
    tokens = word_tokenize(text)
    tokens = [w.lower() for w in tokens]
    table = str.maketrans('', '', string.punctuation)
    stripped = [w.translate(table) for w in tokens]
    words = [word for word in stripped if word.isalpha()]
    stop_words = stopwords.words('english')
    new_stop_words = ['one','came', 'come', 'upon', 'made','though', 'indeed', 'left', 'yet', 'without'
                 'thus','therefore', 'another', 'much', 'many','said', 'either', 'two', 'upon', 'would',
                     'around', 'without', 'when', 'also', 'could', 'sent', 'notwithstanding', 'hence', 'thus']
    stop_words.extend(new_stop_words)
    words = [w for w in words if not w in stop_words]
    new_text = ' '.join(words)
    plt.figure(figsize=(16, 7))
    fd = nltk.FreqDist(words)
    fd.plot(40,title = "40 Most Frequent Words", cumulative=False)
    #print(new_text[:500]) uncomment if you want to print some text
    out_f.write(new_text)
fd.most_common(40)
bigrams = nltk.bigrams(words)
freq_bigrams = nltk.FreqDist(bigrams)

plt.figure(figsize=(13, 7))
freq_bigrams.plot(20)
freq_bigrams.most_common(20)
trigrams = nltk.trigrams(words)
freq_trigrams = nltk.FreqDist(trigrams)

plt.figure(figsize=(13, 7))
freq_trigrams.plot(20)
freq_trigrams.most_common(20)
unique = set(words)
print("The tokenized text is {} words long, has {} unique words and {} letters on average".format
      (len(words), len(unique), round(sum(len(word) for word in words)/len(words),2)))
from nltk.stem.porter import PorterStemmer
with open(fpath + '/Plutarch_tokens.txt') as f, open(fpath + '/Plutarch_stem.txt', 'w') as out_f:
    text = f.read()
    tokens = word_tokenize(text)
    porter = PorterStemmer()
    stemmed = [porter.stem(word) for word in tokens]
    print(stemmed[:100])
    new_stem_text = ' '.join(stemmed)
    fd_stemmed = nltk.FreqDist(stemmed)
    out_f.write(new_stem_text)
unique_stemmed = set(stemmed)
print("The stemmed text is {} words long, has {} unique words and {} letters on average".format
      (len(stemmed), len(unique_stemmed), round(sum(len(word) for word in stemmed)/len(stemmed),2)))
nltk.download('wordnet') #need if using Google Colab
from nltk.stem import WordNetLemmatizer

with open(fpath + '/Plutarch_tokens.txt') as f, open(fpath + '/Plutarch_lemma.txt', 'w') as out_f:
    text = f.read()
    tokens = word_tokenize(text)
    lemma = WordNetLemmatizer()
    lemmed = [lemma.lemmatize(word) for word in tokens]
    print(lemmed[:100])
    new_lem_text = ' '.join(lemmed)
    out_f.write(new_lem_text)
unique_lemmed = set(lemmed)
print("The lemmatized text is {} words long, has {} unique words and {} letters on average".format
      (len(lemmed), len(unique_lemmed), round(sum(len(word) for word in lemmed)/len(lemmed),2)))<jupyter_output><empty_output><jupyter_text>Unique words will help us determining the dictionary size in embeddings Let's convert the into a word cloud! The wordcloud library has its own list of stop words, so we could inspect it - but here we'll be using the previously defined stop words (NLTK's stop words plus some additional ones indicated above); technically, the stop words are not even needed in this  case because the text files being used had already been cleaned.<jupyter_code>#from wordcloud import STOPWORDS
#print(STOPWORDS)

# In case adding stop words is needed, it is as simple as:
#stopwords = set(STOPWORDS)
#stopwords.add("additional_stopword")
import pandas as pd
from wordcloud import WordCloud
import matplotlib.pyplot as plt  
from os import path

d = path.dirname(__file__) if "__file__" in locals() else os.getcwd()

text_standard = open(fpath + '/Plutarch_tokens.txt', 'rt').read()

wc = WordCloud(stopwords=stop_words,
                      max_font_size=200,
                      width=2500,
                      height=2000,
                      max_words=4000,
                      random_state=44,
                     ).generate(text_standard)

wc.to_file(path.join(d, "wc.png")) #comment out if no need to store the png

fig = plt.figure().set_size_inches(18, 18)
plt.imshow(wc, interpolation="bilinear")
plt.axis("off")
plt.title("Plutarch's Lives", fontsize= 20)
plt.show()<jupyter_output><empty_output><jupyter_text>Now let's use the stemmed text and a more artistic way of creating the word cloud<jupyter_code>import numpy as np
from PIL import Image

roman_mask = np.array(Image.open(path.join(d, 'roman7.png')))
text_stem = open(fpath + '/Plutarch_stem.txt', 'rt').read()

wc = WordCloud(stopwords=stop_words, 
               max_font_size=200,
               width=5000,
               height=4000,
               background_color="white", 
               max_words=4000, 
               mask=roman_mask,
               contour_width=3, 
               contour_color='steelblue',
               random_state = 64).generate(text_stem)

wc.to_file(path.join(d, "wc_stemmed.png")) #comment out if no need to store the png

fig = plt.figure().set_size_inches(18, 18)
plt.imshow(wc, interpolation='bilinear')
plt.axis("off")
plt.title("Plutarch's Lives (Stemmed)", fontsize= 20)
plt.show()
roman_mask = np.array(Image.open(path.join(d, 'roman8.png')))
text_lemma = open(fpath + '/Plutarch_lemma.txt', 'rt').read()

wc = WordCloud(stopwords=stop_words, 
               max_font_size=200,
               width=5000,
               height=4000,
               background_color="black", 
               max_words=4000, 
               mask=roman_mask,
               contour_width=3, 
               contour_color='red',
               random_state = 64).generate(text_lemma)

wc.to_file(path.join(d, "wc_lemmed.png")) #comment out if no need to store the png

fig = plt.figure().set_size_inches(18, 18)
plt.imshow(wc, interpolation='bilinear')
plt.axis("off")
plt.title("Plutarch's Lives (Lemmatized)", fontsize= 20)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1207426376440461, 'fraction_numerical': 0.014980793854033291, 'mean_word_length': 3.918765743073048, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '374821', 'license': 'Apache-2.0', 'n_tokens_mistral': 2670, 'n_tokens_neox': 2475, 'n_words': 734, 'path': 'RePlutarch_ExplorePub.ipynb', 'repo_name': 'bromic007/TextExplore', 'size': '7823'}