<jupyter_start><jupyter_text>Sentiment Analysis, V1<jupyter_code>import nltk
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from nltk.tokenize import word_tokenize
from nltk.probability import FreqDist
import matplotlib.pyplot as plt
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.tokenize import sent_tokenize, word_tokenize
import os<jupyter_output><empty_output><jupyter_text>--- PRE-PROCESSING TOKENIZATION Intro to tokenizationTokenization is simply breaking text down into "tokens" which, in this case, is words!Here is a sample sentence. It can have punctuation, CAPS!, etc.<jupyter_code>example_text = "Kendra loves cats. In fact, she has TEN cats. If she didn't have a house, a husband and a graduate degree in data science, she'd be a cat lady!"
tokenized_example = word_tokenize(example_text)
print(tokenized_example)<jupyter_output>['Kendra', 'loves', 'cats', '.', 'In', 'fact', ',', 'she', 'has', 'TEN', 'cats', '.', 'If', 'she', 'did', "n't", 'have', 'a', 'house', ',', 'a', 'husband', 'and', 'a', 'graduate', 'degree', 'in', 'data', 'science', ',', 'she', "'d", 'be', 'a', 'cat', 'lady', '!']<jupyter_text>FREQUENCY DISTRIBUTIONSAfter turning the words into tokens, we can start to treat them a dataFor example, we can use a FREQUENCY DISTRIBUTION to count the words<jupyter_code>fdist = FreqDist(tokenized_example)
print(fdist.items())
# We can use the same Frequency Distribution to find the most common word
print(fdist.most_common(1))
# We can use the Frequency Distribution to find the frequency of specific words
print(fdist.freq("cat")) 
# And we can use the Frequency Distribution to graph our sentence by word freqency
fdist.N()
fdist.plot(30,cumulative=False)
plt.show()<jupyter_output><empty_output><jupyter_text>But... the word "a" doesn't really tell us much about that sentence, does it?Good note! "a" is what is considered to be a STOPWORD  STOPWORDS<jupyter_code>stop_words = set(stopwords.words("english"))
print(stop_words)<jupyter_output>{"haven't", 'why', 'so', "mightn't", 'your', "won't", 'm', 'for', "isn't", 'between', 'this', 'didn', "you're", 'ain', 'not', "it's", 'itself', 'what', 'some', 'now', 'll', 'by', 'but', 'nor', 'ourselves', 'those', 'yours', 'of', 'she', 'to', 'doesn', 'until', "needn't", 'herself', 'is', 'our', 'does', 'then', 'from', 'haven', "hadn't", 'or', "she's", 'over', "didn't", 'are', 'will', "shan't", 'very', 'too', 'had', 'off', 'through', 'he', 're', 'under', 'myself', 'when', 'don', 'each', 'be', 'ours', 'most', 'o', 'y', 'am', 'hers', 'which', 'can', 'yourselves', 'wasn', 'd', 'there', "shouldn't", 'wouldn', 'whom', 'into', 'an', 'in', 'a', 'just', 'you', 'them', "that'll", 'theirs', "weren't", 'they', 'it', 'been', 'having', 'that', 'themselves', 'and', 'about', 'further', "wouldn't", "you'll", 'as', 'my', "wasn't", 'needn', "doesn't", 'its', 'won', 'again', 'where', 'mustn', 'than', "you'd", 'both', "should've", 'weren', 'being', 'shouldn', 'me', 'ma', 'were', 'hasn', "hasn't", 'more', '[...]<jupyter_text>REMOVING STOPWORDS<jupyter_code>filtered_text=[]
for w in tokenized_example:
    if w not in stop_words:
        filtered_text.append(w)
print("Tokenized text:",tokenized_example)
print("Filterd text:",filtered_text)
fdist_filtered = FreqDist(filtered_text)
fdist_filtered.plot(30,cumulative=False)
plt.show()<jupyter_output><empty_output><jupyter_text>But wait, doesn't the word "cat" appear more than 2 times? Oh! That's because we have "cat" AND "cats" which the computer is counting as two different words!Introducing...  STEMMING<jupyter_code>ps = PorterStemmer()
stemmed_words=[]
for w in filtered_text:
    stemmed_words.append(ps.stem(w))
fdist_stemmed = FreqDist(stemmed_words)
fdist_stemmed.plot(30,cumulative=False)
plt.show()<jupyter_output><empty_output><jupyter_text>LEMMATIZATIONAlso, Lemmatization! A side quest!Lemmatization reduces words to their base word<jupyter_code>from nltk.stem.wordnet import WordNetLemmatizer
lem = WordNetLemmatizer() 
from nltk.stem.porter import PorterStemmer
stem = PorterStemmer()
word = "carrying"
# 'v' because carrying is a verb
print("Lemmatized Word:",lem.lemmatize(word,"v"))
print("Stemmed Word:",stem.stem(word))
word = "better"
# 'a' because better is an adj
print("Lemmatized Word:",lem.lemmatize(word,"a"))
print("Stemmed Word:",stem.stem(word))<jupyter_output>Lemmatized Word: good
Stemmed Word: better<jupyter_text>PART-OF-SPEECH (POS) TaggingAnother side quest!!This helps identify the grammatical group -- Noun, pronoun, adjective, verb, adverb etc<jupyter_code>sent = "Kendra loves cats. In fact, she has TEN cats. If she didn't have a house, a husband and a graduate degree in data science, she'd be a cat lady!"
Mytokens=nltk.word_tokenize(sent)
MyTAGS = nltk.pos_tag(Mytokens)
print(MyTAGS)<jupyter_output>[('Kendra', 'NNP'), ('loves', 'VBZ'), ('cats', 'NNS'), ('.', '.'), ('In', 'IN'), ('fact', 'NN'), (',', ','), ('she', 'PRP'), ('has', 'VBZ'), ('TEN', 'VBN'), ('cats', 'NNS'), ('.', '.'), ('If', 'IN'), ('she', 'PRP'), ('did', 'VBD'), ("n't", 'RB'), ('have', 'VB'), ('a', 'DT'), ('house', 'NN'), (',', ','), ('a', 'DT'), ('husband', 'NN'), ('and', 'CC'), ('a', 'DT'), ('graduate', 'NN'), ('degree', 'NN'), ('in', 'IN'), ('data', 'NNS'), ('science', 'NN'), (',', ','), ('she', 'PRP'), ("'d", 'MD'), ('be', 'VB'), ('a', 'DT'), ('cat', 'JJ'), ('lady', 'NN'), ('!', '.')]<jupyter_text>--- MACHINE LEARNING Count Vectorizer TESTING SMALL FILES FIRST<jupyter_code>import sklearn
from sklearn.feature_extraction.text import CountVectorizer
all_file_names = []
path="/Users/danielcaraway/Documents/IST_736_TextMining/SmallTextDocsK"
FileNameList=os.listdir(path)

# METHOD A:
# Getting a list of filenames
ListOfCompleteFiles=[]
for name in os.listdir(path):
    next=path+ "/" + name
    ListOfCompleteFiles.append(next)

# METHOD B:
# Getting the text from the filenames
AllText_AllFiles=[]
for file in ListOfCompleteFiles:
    f=open(file)
    content=f.read()
    AllText_AllFiles.append(content)
    f.close()
    
# AllText_AllFiles[0]
# METHOD A (with filenames)
vec_filename = CountVectorizer(input='filename')
transformed_files = vec_filename.fit_transform(ListOfCompleteFiles)
files_feature_names = vec_filename.get_feature_names()
transformed_files_df = pd.DataFrame(transformed_files.toarray(),columns=files_feature_names)
transformed_files_df
# METHOD B (with all text from files)
vec_content = CountVectorizer(input='content')
transformed_text = vec_content.fit_transform(AllText_AllFiles)
content_feature_names = vec_content.get_feature_names()
transformed_text_df = pd.DataFrame(transformed_text.toarray(),columns=content_feature_names)
transformed_text_df<jupyter_output><empty_output><jupyter_text>TESTING SMALL DATASET<jupyter_code>vec_removing_sw = CountVectorizer(input='filename',
                        analyzer = 'word',
                        stop_words='english',
                        token_pattern='(?u)[a-zA-Z]+')
# POSITIVE FILES
path="/Users/danielcaraway/Documents/IST_736_TextMining/AI_POS"
pos_files=[]
for name in os.listdir(path):
    next=path+ "/" + name
    pos_files.append(next)
pos_transformed = vec_removing_sw.fit_transform(pos_files)
pos_transformed_feature_names = vec_removing_sw.get_feature_names()
pos_transformed_df = pd.DataFrame(pos_transformed.toarray(), columns = pos_transformed_feature_names)
pos_transformed_df['PoN'] = 'p'
pos_transformed_df
# NEGATIVE FILES
path="/Users/danielcaraway/Documents/IST_736_TextMining/AI_NEG"
neg_files=[]
for name in os.listdir(path):
    next=path+ "/" + name
    neg_files.append(next)
neg_transformed = vec_removing_sw.fit_transform(neg_files)
neg_transformed_feature_names = vec_removing_sw.get_feature_names()
neg_transformed_df = pd.DataFrame(neg_transformed.toarray(), columns = neg_transformed_feature_names)
neg_transformed_df['PoN'] = 'n'
neg_transformed_df
combined = pos_transformed_df.append(neg_transformed_df, sort=False)
combined
combined = combined.fillna(0)
combined<jupyter_output><empty_output><jupyter_text>Prepare Train and Test<jupyter_code>from IPython.display import display, HTML
display(combined)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1741579914268218, 'fraction_numerical': 0.002449479485609308, 'mean_word_length': 6.859480269489894, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '627725', 'license': 'MIT', 'n_tokens_mistral': 3020, 'n_tokens_neox': 2855, 'n_words': 710, 'path': 'assets/all_html/2019_10_05_HW1.ipynb', 'repo_name': 'dskw1/dskw1.github.io', 'size': '8165'}