<jupyter_start><jupyter_text>Analyze Language and Sentiment of The Picture of <NAME><jupyter_code>import os
from os import listdir
os.listdir()<jupyter_output><empty_output><jupyter_text>1. Load text2. Preprocess* Tokenize* Remove stop words3<jupyter_code>import bs4 as bs
import urllib.request
import re
import nltk
filename='dorian_gray.txt'
file = open(filename,'rt')
text = file.read()
file.close()
#Parse novel
parsed_novel = bs.BeautifulSoup(text, 'lxml')

paragraphs = parsed_novel.find_all('p')

#Join all paragraphs together in variable
novel_text =''

for p in paragraphs:
    novel_text += p.text
#Cleaning text
processed_novel= novel_text.lower()
processed_novel = re.sub('[^a-zA-Z]', ' ', processed_novel)
processed_novel = re.sub(r'\s+',' ',processed_novel)

#preparing the dataset
all_sentences = nltk.sent_tokenize(processed_novel)
all_words = [nltk.word_tokenize(sent) for sent in all_sentences]

#remove stop words
from nltk.corpus import stopwords
for i in range(len(all_words)):
    all_words[i] = [w for w in all_words[i] if w not in stopwords.words('english')]
from gensim.models import Word2Vec

word2vec = Word2Vec(all_words, min_count = 10)
vocabulary = word2vec.wv.vocab
print(vocabulary)
sim_words = word2vec.wv.most_similar('art')
sim_words
words = text.split()
print(words[:100])
import string
print(string.punctuation)
table = str.maketrans(' ', ' ', string.punctuation)
stripped = [w.translate(table) for w in words]
print(stripped[:100])
import nltk
nltk.download()
from nltk.corpus import stopwords
stop_words = stopwords.words('english')
filename = 'dorian_gray.txt'
file = open(filename, 'rt')
text = file.read()
file.close()

#split into words
from nltk import word_tokenize
tokens = word_tokenize(text)

#filter
from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
filter_text = [w for w in tokens if not w in stop_words]

import string
table = str.maketrans(' ', ' ', string.punctuation)
stripped = [w.translate(table) for w in filter_text]
print(stripped[:100])

from sklearn.feature_extraction.text import CountVectorizer
import nltk
dorian_vec = CountVectorizer(min_df=2, tokenizer = nltk.word_tokenize, stop_words = 'english')
x_counts = dorian_vec.fit_transform(stripped)
X_binary = x_counts>0
import nltk
nltk.download('vader_lexicon')
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()
well = sid.polarity_scores(stripped)
from sklearn.naive_bayes import MultinomialNB
from sklearn.naive_bayes import BernoulliNB
stripped<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10989867498051442, 'fraction_numerical': 0.00857365549493375, 'mean_word_length': 6.708708708708708, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '89536', 'license': 'MIT', 'n_tokens_mistral': 955, 'n_tokens_neox': 918, 'n_words': 222, 'path': 'The_Language_of_Dorian_Gray.ipynb', 'repo_name': 'JesShea/language_of_dorian_gray', 'size': '2571'}