<jupyter_start><jupyter_text>Importing Data Data was scraped from the Amazon website for Charge 2<jupyter_code>df = pd.read_csv('charge2.csv', parse_dates =['date'])
df.head()
df = df.drop(columns=['web-scraper-order','web-scraper-start-url'])
df['stars'] = df.stars.apply(lambda x: x.split(' ')[0])
df.head()
df['length'] = df.review.apply(lambda x: len(x))
df['year'] = df.date.apply(lambda x: x.year)
df['stars'] = pd.to_numeric(df.stars)
df.head()
fig, ax = plt.subplots(figsize = (14,8))
sns.boxplot(df.stars, df.length)
plt.ylim(-20, 2000)
fig, ax = plt.subplots(figsize = (14,8))
df.groupby(['date'])['stars'].mean().plot(marker='.',linestyle='none')
fig, ax = plt.subplots(figsize = (14,8))
plt.plot(df.date, df.stars, alpha=0.02, marker='.', linestyle='none', markersize=40)
df.shape
fig, ax = plt.subplots(figsize = (14,8))
df.groupby('stars').size().plot(kind='bar', color='orange', alpha=0.7)
plt.xlabel('Star Rating')
plt.ylabel('Number of Ratings')
plt.title('Amazon Reviews for Fitbit Charge 2', fontsize=14)<jupyter_output><empty_output><jupyter_text>NLP<jupyter_code>import gensim
import spacy
from nltk import FreqDist
from gensim.models import LdaModel, phrases
from gensim.models.wrappers import LdaMallet
from gensim.corpora import Dictionary
import pyLDAvis.gensim

import os, re, operator, warnings
warnings.filterwarnings('ignore')  # Let's not pay heed to them right now<jupyter_output><empty_output><jupyter_text>SpaCy<jupyter_code>terms = [u"heart rate", u"gps", u"sync", u"syncing", u"design",
         u"charging", u"charger", u"battery", u"quality", 
         u"screen", u"service", u"sleep", u"alarm", u"reminder"]
#$ python -m spacy download en_core_web_md (run in shell if not installed)
nlp = spacy.load('en_core_web_lg')
text = df.review[12]
print(text)
# Process the text
doc = nlp(text)
for token in doc:
    if token.lemma_ in terms:
        print("{0}/{1} <--{2}-- {3}/{4}".format(
        token.text, token.tag_, token.dep_, token.head.text, token.head.tag_))
for sentence in doc.sents:
    chunk = []
    for token in sentence:     
        if token.lemma_ in terms:
            print(sentence)
            print('---')
            print("{0}/{1} <--{2}-- {3}/{4}".format(
                  token.text, token.tag_, token.dep_, token.head.text, token.head.tag_))
            print('---')           
            for child in sentence.root.children:
                chunk.append(child.text)
                print(f'child: {child}')
from spacy import displacy

# In[2]:
doc = nlp(u"i am very disapointed especially about the heart rate tracker")
displacy.render(doc, style="dep", jupyter=True, options={'distance':90})
for sentence in doc.sents:
    print(sentence)
    print('--------------------')
'''#stemming
from nltk.stem.snowball import SnowballStemmer
stemmer = SnowballStemmer(language='english')'''
#A very slow way of extracting search terms
'''for term in terms:    
    for i in doc:
        sims = nlp(stemmer.stem(term))[0].similarity(nlp(stemmer.stem(str(i))))
        if sims > 0.7:
            print(sims, i)'''<jupyter_output><empty_output><jupyter_text>Phrase Matcher<jupyter_code>from spacy.matcher import PhraseMatcher
terms = [u"heart rate", u"gps", u"sync", u"syncing", u"design",
         u"charging", u"charger", u"battery", u"quality", 
         u"screen", u"service", u"sleep", u"alarm", u"reminder"]
def phrase_matcher(review): 
    matched_list = []
    matcher = PhraseMatcher(nlp.vocab)
    # Only run nlp.make_doc to speed things up
    patterns = [nlp.make_doc(text) for text in terms]
    matcher.add("TerminologyList", None, *patterns)
    matches = matcher(review)
    for match_id, start, end in matches:
        span = review[start:end]
        matched_list.append(span.text)
    return list(set(matched_list))<jupyter_output><empty_output><jupyter_text>Sentiment Analysis - TextBlob, Afinn, NLTK+Vader<jupyter_code>#TEXTBLOB
from textblob import TextBlob
#AFINN
from afinn import Afinn
af = Afinn()
#NLTK
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import nltk
nltk.download('vader_lexicon')
sid = SentimentIntensityAnalyzer()<jupyter_output>[nltk_data] Downloading package vader_lexicon to
[nltk_data]     /Users/dmitriykats/nltk_data...
[nltk_data]   Package vader_lexicon is already up-to-date!<jupyter_text>Phrase count in new DataFrame<jupyter_code>#create a dataframe with our phrases as columns, to keep count of sentiment
phrase_df = pd.DataFrame(columns=terms)
def get_sentiment(input_string): 
    '''
    Takes a string (sentence), and returns an averaged, three-model 
    sentiment score, from -1 to 1. Where -1 is negative sentiment, 0 is neutral,
    and +1 is positive sentiment. 
    '''
    #calculate sentiment score with TextBlob, Afinn, NLTK+Vader
    txt_blob = TextBlob(input_string.text).sentiment[0]
    a_fin = af.score(input_string.text)
    nltk_vader = sid.polarity_scores(input_string.text).get('compound')
    
    #TextBlob
    if txt_blob > 0.1:
        blob = 1
    elif txt_blob == 0 and txt_blob <= 0.1:
        blob = 0
    else: blob = -1

    #afinn
    if a_fin > 1:
         fin = 1
    elif a_fin == 0 and a_fin <= 1:
        fin = 0
    else: fin = -1

    #NLTK-Vader
    if nltk_vader > 0.1:
        vader = 1
    elif nltk_vader == 0 and nltk_vader <= 0.1:
        vader = 0
    else: vader = -1
   
    #Averaged return
    avg_score = (blob + fin + vader) / 3
    return avg_score      
#need SpaCy lg dictionary and TextBlob

def review_sent(review):
    '''
    for a given review, returns dictionary of matched phrases : sentiment
    '''
    sent_dict = {}
    count_df = pd.DataFrame(columns=['phrase','sentiment'])
    doc = nlp(review)
    
    for sentence in doc.sents:
        matched_phrases = phrase_matcher(nlp(str(sentence)))
        phrases = ', '.join(matched_phrases)
        
        #Get Sentiment from models
        sentence_sentiment = get_sentiment(sentence)

        #check if more than one phrase found by phrase matcher
        if len(matched_phrases) > 1:
            #add each phrase of the given sentence to the dataframe
            for item in matched_phrases:
                count_df.loc[-1] = [item, sentence_sentiment]   
                count_df.index = count_df.index + 1
                count_df = count_df.sort_index()
        else:    
            count_df.loc[-1] = [phrases, sentence_sentiment]   
            count_df.index = count_df.index + 1
            count_df = count_df.sort_index()
          
    grouped_df = count_df.groupby('phrase').sum()
    grouped_dict = grouped_df.to_dict()
    sentimen_dict = grouped_dict['sentiment']
    return sentimen_dict
df['sentiment'] = df.review.apply(lambda x: review_sent(x))
df_concat = pd.concat([df, phrase_df], axis=1)
for i in phrase_df.columns:
    df_concat[i] = df_concat.apply(lambda x: x.sentiment.get(i), axis=1)
new_df = df_concat.iloc[:, 12:]
df_concat.groupby('stars').sum().iloc[:, 2:]
new_df.head()
fig, ax = plt.subplots(figsize = (14,8))
new_df.count().sort_values().plot(kind='barh', color='orange', alpha=0.7)
plt.title('Total Count of Topics Mentioned in Reviews', fontsize=16)
plt
fig, ax = plt.subplots(figsize = (14,8))
new_df.sum().sort_values().plot(kind='barh', color='orange', alpha=0.7)
plt.title('Feature Sentiment from Charge2 Amazon Reviews', fontsize=16)
plt.xlabel('Count of Negative and Positive Comments', fontsize=12)
sns.set(font_scale=1.05)
fig, ax = plt.subplots(figsize = (20,6))
sns.heatmap(df_concat.groupby('stars').sum().iloc[:, 2:], vmin=-120, vmax=120,
            annot=True, linewidths=0.3, cmap='YlGnBu', 
            cbar_kws={'label': '<--- negative sentiment --- positive sentiment --->'})
ax.invert_yaxis()
plt.title('Net Sentiment Score of Topic by Start Rating', fontsize=16)
plt.xlabel('Topic', fontsize=14)
plt.ylabel('Amazon Star Rating', fontsize=14)
fig.savefig('Net_Sentiment_Scores.png')<jupyter_output><empty_output><jupyter_text>Phrase count in new DataFrame (TextBlob only)<jupyter_code>#create a dataframe with our phrases as columns, to keep count of sentiment
phrase_df = pd.DataFrame(columns=terms)
#need SpaCy lg dictionary and TextBlob

def review_sent(review):
    '''
    for a given review, returns dictionary of matched phrases : sentiment
    '''
    sent_dict = {}
    count_df = pd.DataFrame(columns=['phrase','sentiment'])
    #create textblob object
    rev = TextBlob(review)
    
    for sentence in rev.sentences:
        matched_phrases = phrase_matcher(nlp(str(sentence)))
        phrases = ', '.join(matched_phrases)
    
        if sentence.sentiment[0] > 0.1:
            sentence_sentiment = 1
        elif sentence.sentiment[0] == 0 and sentence.sentiment[0] <= 0.1:
            sentence_sentiment = 0
        else:
            sentence_sentiment = -1
        
        #check if more than one phrase found by phrase matcher
        if len(matched_phrases) > 1:
            #add each phrase of the given sentence to the dataframe
            for item in matched_phrases:
                count_df.loc[-1] = [item, sentence_sentiment]   
                count_df.index = count_df.index + 1
                count_df = count_df.sort_index()
        else:    
            count_df.loc[-1] = [phrases, sentence_sentiment]   
            count_df.index = count_df.index + 1
            count_df = count_df.sort_index()
          
    grouped_df = count_df.groupby('phrase').sum()
    grouped_dict = grouped_df.to_dict()
    sentimen_dict = grouped_dict['sentiment']
    return sentimen_dict
df['sentiment'] = df.review.apply(lambda x: review_sent(x))
df_concat = pd.concat([df, phrase_df], axis=1)
for i in phrase_df.columns:
    df_concat[i] = df_concat.apply(lambda x: x.sentiment.get(i), axis=1)
new_df = df_concat.iloc[:, 12:]
new_df.head()
fig, ax = plt.subplots(figsize = (14,8))
new_df.count().sort_values().plot(kind='barh', color='orange', alpha=0.7)

fig, ax = plt.subplots(figsize = (14,8))
new_df.sum().sort_values().plot(kind='barh', color='orange', alpha=0.7)
plt.title('Feature Sentiment from Charge2 Amazon Reviews', fontsize=16)
plt.xlabel('Count of Negative and Positive Comments', fontsize=12)<jupyter_output><empty_output><jupyter_text>Phrase count in new DataFrame (AFFIN only)<jupyter_code>from afinn import Afinn
af = Afinn()
#need SpaCy lg dictionary and AFFIN

def review_sent(review):
    '''
    for a given review, returns dictionary of matched phrases : sentiment
    '''
    sent_dict = {}
    count_df = pd.DataFrame(columns=['phrase','sentiment'])
    #create textblob object
    rev = TextBlob(review)
    
    for sentence in rev.sentences:
        matched_phrases = phrase_matcher(nlp(str(sentence)))
        phrases = ', '.join(matched_phrases)
    
        if af.score(str(sentence)) > 1:
            sentence_sentiment = 1
        elif af.score(str(sentence)) == 0 and af.score(str(sentence)) <= 1:
            sentence_sentiment = 0
        else:
            sentence_sentiment = -1
        
        #check if more than one phrase found by phrase matcher
        if len(matched_phrases) > 1:
            #add each phrase of the given sentence to the dataframe
            for item in matched_phrases:
                count_df.loc[-1] = [item, sentence_sentiment]   
                count_df.index = count_df.index + 1
                count_df = count_df.sort_index()
        else:    
            count_df.loc[-1] = [phrases, sentence_sentiment]   
            count_df.index = count_df.index + 1
            count_df = count_df.sort_index()
          
    grouped_df = count_df.groupby('phrase').sum()
    grouped_dict = grouped_df.to_dict()
    sentimen_dict = grouped_dict['sentiment']
    return sentimen_dict
df['sentiment'] = df.review.apply(lambda x: review_sent(x))
df_concat = pd.concat([df, phrase_df], axis=1)
for i in phrase_df.columns:
    df_concat[i] = df_concat.apply(lambda x: x.sentiment.get(i), axis=1)
new_df_afinn = df_concat.iloc[:, 12:]
new_df_afinn.head()
fig, ax = plt.subplots(figsize = (14,8))
new_df_afinn.count().sort_values().plot(kind='barh', color='orange', alpha=0.7)
fig, ax = plt.subplots(figsize = (14,8))
new_df_afinn.sum().sort_values().plot(kind='barh', color='orange', alpha=0.7)
plt.title('Feature Sentiment from Charge2 Amazon Reviews', fontsize=16)
plt.xlabel('Count of Negative and Positive Comments', fontsize=12)<jupyter_output><empty_output><jupyter_text>Phrase count in new DataFrame (NLTK only)<jupyter_code>from nltk.sentiment.vader import SentimentIntensityAnalyzer
import nltk
nltk.download('vader_lexicon')
#need SpaCy lg dictionary 

def review_sent(review):
    '''
    for a given review, returns dictionary of matched phrases : sentiment
    '''
    sent_dict = {}
    count_df = pd.DataFrame(columns=['phrase','sentiment'])
    #create textblob object
    rev = TextBlob(review)
    
    for sentence in rev.sentences:
        matched_phrases = phrase_matcher(nlp(str(sentence)))
        phrases = ', '.join(matched_phrases)
    
        if sid.polarity_scores(str(sentence)).get('compound') > 0.1:
            sentence_sentiment = 1
        elif sid.polarity_scores(str(sentence)).get('compound') == 0 \
            and sid.polarity_scores(str(sentence)).get('compound') <= 0.1:
            sentence_sentiment = 0
        else:
            sentence_sentiment = -1
        
        #check if more than one phrase found by phrase matcher
        if len(matched_phrases) > 1:
            #add each phrase of the given sentence to the dataframe
            for item in matched_phrases:
                count_df.loc[-1] = [item, sentence_sentiment]   
                count_df.index = count_df.index + 1
                count_df = count_df.sort_index()
        else:    
            count_df.loc[-1] = [phrases, sentence_sentiment]   
            count_df.index = count_df.index + 1
            count_df = count_df.sort_index()
          
    grouped_df = count_df.groupby('phrase').sum()
    grouped_dict = grouped_df.to_dict()
    sentimen_dict = grouped_dict['sentiment']
    return sentimen_dict
df['sentiment'] = df.review.apply(lambda x: review_sent(x))
df_concat = pd.concat([df, phrase_df], axis=1)
for i in phrase_df.columns:
    df_concat[i] = df_concat.apply(lambda x: x.sentiment.get(i), axis=1)
new_df_nltk = df_concat.iloc[:, 12:]
new_df_nltk.head()
fig, ax = plt.subplots(figsize = (14,8))
new_df_nltk.count().sort_values().plot(kind='barh', color='orange', alpha=0.7)
fig, ax = plt.subplots(figsize = (14,8))
new_df_nltk.sum().sort_values().plot(kind='barh', color='orange', alpha=0.7)
plt.title('Feature Sentiment from Charge2 Amazon Reviews', fontsize=16)
plt.xlabel('Count of Negative and Positive Comments', fontsize=12)<jupyter_output><empty_output><jupyter_text>Sentiment Analyis with StanfordCoreNLP<jupyter_code>from pycorenlp import StanfordCoreNLP
stan_nlp = StanfordCoreNLP('http://localhost:9000')
text = df.review[19]
print(text)
reviews = text
result = stan_nlp.annotate(text,
                   properties={
                       'annotators': 'sentiment, ner, pos',
                       'outputFormat': 'json',
                       'timeout': 5000,
                   })
for s in result["sentences"]:
    print("{}: '{}': {} (Sentiment Value) {} (Sentiment)".format(
        s["index"],
        " ".join([t["word"] for t in s["tokens"]]),
        s["sentimentValue"], s["sentiment"]))
sid.polarity_scores('The sleep tracker breaks down sleep by stages which is great .').get('compound')<jupyter_output><empty_output><jupyter_text>Training Keras Deep Sentiment Model<jupyter_code>df['aspect_category'] = df.review.apply(lambda x: sorted(phrase_matcher(nlp(x))))
df.head()
df['aspect_cat_string'] = df.aspect_category.apply(lambda x: ', '.join(x))
df['review'] = df.review.str.lower()

aspect_terms = []
for review in nlp.pipe(df.review):
    chunks = [(chunk.root.text) for chunk in review.noun_chunks if chunk.root.pos_ == 'NOUN']
    aspect_terms.append(' '.join(chunks))
df['aspect_terms'] = aspect_terms
df.head(5)
from keras.models import load_model
from keras.models import Sequential
from keras.layers import Dense, Activation

aspect_categories_model = Sequential()
aspect_categories_model.add(Dense(512, input_shape=(6000,), activation='relu'))
aspect_categories_model.add(Dense(205, activation='softmax'))
aspect_categories_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
from keras.preprocessing.text import Tokenizer

vocab_size = 6000 # We set a maximum size for the vocabulary
tokenizer = Tokenizer(num_words=vocab_size)
tokenizer.fit_on_texts(df.review)
aspect_tokenized = pd.DataFrame(tokenizer.texts_to_matrix(df.aspect_terms))
from sklearn.preprocessing import LabelEncoder
from keras.utils import to_categorical

label_encoder = LabelEncoder()
integer_category = label_encoder.fit_transform(df.aspect_cat_string)
dummy_category = to_categorical(integer_category)
aspect_categories_model.fit(aspect_tokenized, dummy_category, epochs=5, verbose=1)
new_review = text

chunks = [(chunk.root.text) for chunk in nlp(new_review).noun_chunks if chunk.root.pos_ == 'NOUN']
new_review_aspect_terms = ' '.join(chunks)
new_review_aspect_tokenized = tokenizer.texts_to_matrix([new_review_aspect_terms])

new_review_category = label_encoder.inverse_transform(aspect_categories_model.predict_classes(new_review_aspect_tokenized))
print(new_review_category)
sentiment_terms = []
for review in nlp.pipe(df['review']):
        if review.is_parsed:
            sentiment_terms.append(' '.join([token.lemma_ for token in review if (not token.is_stop and not token.is_punct and (token.pos_ == "ADJ" or token.pos_ == "VERB"))]))
        else:
            sentiment_terms.append('')  
df['sentiment_terms'] = sentiment_terms
df.head()
sentiment_model = Sequential()
sentiment_model.add(Dense(512, input_shape=(6000,), activation='relu'))
sentiment_model.add(Dense(3, activation='softmax'))
sentiment_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
sentiment_tokenized = pd.DataFrame(tokenizer.texts_to_matrix(df.sentiment_terms))
def sentiment_return(rating):
    sentiment = 'positive'
    if rating == 3:
        sentiment = 'neutral'
    elif rating < 3:
        sentiment = 'negative'
    return sentiment
df['sentiment'] = df.stars.apply(lambda x: sentiment_return(x))
df.head()
label_encoder_2 = LabelEncoder()
integer_sentiment = label_encoder_2.fit_transform(df.sentiment)
dummy_sentiment = to_categorical(integer_sentiment)
sentiment_model.fit(sentiment_tokenized, dummy_sentiment, epochs=10, verbose=1)
new_review = 'hate everything, and the watch does not work, sync is bad, sreen is dead'

chunks = [(chunk.root.text) for chunk in nlp(new_review).noun_chunks if chunk.root.pos_ == 'NOUN']
new_review_aspect_terms = ' '.join(chunks)
new_review_aspect_tokenized = tokenizer.texts_to_matrix([new_review_aspect_terms])

new_review_category = label_encoder_2.inverse_transform(sentiment_model.predict_classes(new_review_aspect_tokenized))
print(new_review_category)
test_reviews = [
    "customer support they tell me to restart the watch. The problem has not been fixed, i'm returing it. Also, the watch does not sync!",
    "Sreen went blank after 3 days. Fitbit stopped syncing after 2 days",
    "Customer service just kept telling me to restart the watch, but could not fix my sync problem. I can't return it. awful support."
]

# Aspect preprocessing
test_reviews = [review.lower() for review in test_reviews]
test_aspect_terms = []
for review in nlp.pipe(test_reviews):
    chunks = [(chunk.root.text) for chunk in review.noun_chunks if chunk.root.pos_ == 'NOUN']
    test_aspect_terms.append(' '.join(chunks))
test_aspect_terms = pd.DataFrame(tokenizer.texts_to_matrix(test_aspect_terms))
                             
# Sentiment preprocessing
test_sentiment_terms = []
for review in nlp.pipe(test_reviews):
        if review.is_parsed:
            test_sentiment_terms.append(' '.join([token.lemma_ for token in review if (not token.is_stop and not token.is_punct and (token.pos_ == "ADJ" or token.pos_ == "VERB"))]))
        else:
            test_sentiment_terms.append('') 
test_sentiment_terms = pd.DataFrame(tokenizer.texts_to_matrix(test_sentiment_terms))

# Models output
test_aspect_categories = label_encoder.inverse_transform(aspect_categories_model.predict_classes(test_aspect_terms))
test_sentiment = label_encoder_2.inverse_transform(sentiment_model.predict_classes(test_sentiment_terms))
for i in range(3):
    print("Review " + str(i+1) + " is expressing a  " + test_sentiment[i] + " opinion about " + test_aspect_categories[i])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1272444918955644, 'fraction_numerical': 0.01315150926914491, 'mean_word_length': 3.780097425191371, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '12898', 'license': 'MIT', 'n_tokens_mistral': 7220, 'n_tokens_neox': 6898, 'n_words': 1545, 'path': 'Charge_2_ABSA.ipynb', 'repo_name': 'dmitriykats1/Fitbit-Amazon-Reviews', 'size': '20606'}