<jupyter_start><jupyter_text>**Project 4 Notebook 3**Start with dataframe containing the story text tokenized into single words, take out stopwords, and vectorize by single words.<jupyter_code>import pandas as pd
import numpy as np
import re
from pprint import pprint
import nltk
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.decomposition import NMF
from sklearn.metrics.pairwise import cosine_similarity
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel
import spacy
#import pyLDAvis
#import pyLDAvis.gensim  # don't skip this
import matplotlib.pyplot as plt
%matplotlib inline
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.ERROR)
#warnings.filterwarnings("ignore",category=DeprecationWarning)
import re
import string
from nltk.util import ngrams
from nltk.tokenize import word_tokenize
from nltk.tokenize import sent_tokenize
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
#set(stopwords.words('english'))<jupyter_output><empty_output><jupyter_text>Read in the .csv file containing the story text tokenized by single words.<jupyter_code>katniss_singlewords = pd.read_csv('Project-4-data/katniss_complete_restart_wtokenized_2_24.csv')
katniss_singlewords.head(1)
nltk.download('stopwords')
stop=stopwords.words('english')
#Now I can try to take out the stopwords.
katniss_singlewords['story_text_without_stopwords'] = katniss_singlewords['story_text_wtokenized'].apply(lambda x: [item for item in x if item not in stop])
katniss_singlewords.head(1)<jupyter_output><empty_output><jupyter_text>That did not work! The text was separated into single letters!<jupyter_code>katniss_singlewords.drop(['story_text_without_stopwords'], axis=1, inplace=True )
katniss_singlewords.head(1)
for story in katniss_singlewords['story_text_wtokenized']:
    for word in story:
        new_story=[]
        word.apply(lambda x: [item for item in x if item not in stop])
        new_story.append(word)
    katniss_quadgrams_nostop_list.append(new_story)
katniss_2 = pd.read_csv('Project-4-data/katniss-no-num-punc-quote.csv')
katniss_2.head(1)
#Replace capital letters with lowercase
alphanumeric = lambda x: re.sub('\w*\d\w*', ' ', x)
punc_lower = lambda x: re.sub('[%s]' % re.escape(string.punctuation), ' ', x.lower()) 
katniss_2['story_text_lower'] = katniss_2.story_text.map(alphanumeric).map(punc_lower)
katniss_2.head(1)
katniss_2.to_csv('katniss-restart-single-lower.csv')
katniss_2.head(1)<jupyter_output><empty_output><jupyter_text>It separated the text into single letters again. Drop that column and try again.<jupyter_code>katniss_2.drop(['story_text_without_stopwords'], axis=1, inplace=True )
katniss_2.head(1)
#tokenize by word
katniss_2['story_text_tokenize'] = katniss_2['story_text_lower'].apply(word_tokenize)
katniss_2.head(1)<jupyter_output><empty_output><jupyter_text>Using .apply(word_tokenize) worked. I'll drop a column to make the file smaller.<jupyter_code>katniss_2.drop(['story_text'], axis=1, inplace=True )
#Now I can try to take out the stopwords.
katniss_2['story_text_without_stopwords'] = katniss_2['story_text_lower'].apply(lambda x: [item for item in x if item not in stop])
katniss_2.head(1)
#that didn't work
katniss_2.drop(['story_text_without_stopwords'], axis=1, inplace=True)
katniss_2.head(1)
#Now I can try to take out the stopwords.
katniss_2['story_text_without_stopwords'] = (katniss_2['story_text_tokenize']).apply(lambda x: [item for item in x if item not in stop])
katniss_2.head(5)
#The extra parenthesis makes all the difference. Save as a .csv
katniss_2.to_csv('katniss2-restart-single-lower-nostop.csv')
katniss_2.head(5)
from nltk.stem.lancaster import LancasterStemmer
st = LancasterStemmer()
new_column=[]
for story in katniss_2['story_text_without_stopwords']:
    new_story=[]
    for word in story:
        new_word=st.stem(word)
        #print(new_word)
        new_story.append(new_word)
    #print(new_story)
    new_column.append(new_story)
#print(new_column)
katniss_2['story_text_stemmed']=new_column
katniss_2.head(2)
katniss_2.to_csv('katniss2-restart-single-lower-nostop-stem.csv')
#try to change the stemmed column to string data type to feed into tfidf vectorizer.
#if it doesn't work, I can reload the .csv file above and try something else.
katniss_2['story_text_stemmed']=katniss_2['story_text_stemmed'].astype(str)
katniss_2.head(1)<jupyter_output><empty_output><jupyter_text>Vectorize using TF-IDF.<jupyter_code>#vectorize
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
corpus=katniss_2['story_text_stemmed']
cv_tfidf = TfidfVectorizer(decode_error='ignore', strip_accents='unicode', min_df=0.005)
X_tfidf = cv_tfidf.fit_transform(corpus).toarray() 
katniss_doc_term=pd.DataFrame(X_tfidf, columns=cv_tfidf.get_feature_names())
katniss_doc_term.head(10)
#need to adjust min_df. No min_df yields 21510 columns, and several words such as "aaaaaaaaaaah" or some such.
#min_df=0.015 reduced the number of columns to 3941.
#min_df=0.01 reduced to 4664 columns
#min_df=0.005 yields 6246 columns. Try topic modeling with this.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10172349194454852, 'fraction_numerical': 0.017234919445485202, 'mean_word_length': 7.064954682779456, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '369432', 'license': 'MIT', 'n_tokens_mistral': 1952, 'n_tokens_neox': 1902, 'n_words': 432, 'path': 'Notebook-3-Project-4-single-words-remove-stops-vectorize.ipynb', 'repo_name': 'sutrofog/Sillman-Metis-Project4', 'size': '5338'}