<jupyter_start><jupyter_text>Reading data from JSON file<jupyter_code>df = pd.read_json('reviews_merged.json')
df<jupyter_output><empty_output><jupyter_text>New Dataframe having only helpful attribute and productID<jupyter_code>helpfulness = df[['helpful','asin']]
helpfulness
df2 = pd.DataFrame(helpfulness)
df2[['helpful','total']] = pd.DataFrame(df2.helpful.values.tolist(), index=df2.index)
dfhelpful = df2[['asin', 'helpful','total']]
dfhelpful
dfNew = df.drop('helpful',1)
frames = [dfNew, dfhelpful]
result = dfNew.join(dfhelpful[['helpful','total']])
result
result.to_csv('reviews.csv', sep=',', header=False,index=False)
result<jupyter_output><empty_output><jupyter_text>Loading csv to SQLite Fetch the score and recommendation summary using SQL queries<jupyter_code>class csvrd(object):
    def csvFile(self):

        self.readFile('reviews.csv')

    def readFile(self, filename):
        conn = sqlite3.connect('amazonReviews.db')
        cur = conn.cursor() 
        cur.execute("""CREATE TABLE IF NOT EXISTS amazonReviews(asin INT,overall INT,reviewText varchar,reviewTime INTEGER, reviewerID varchar,reviewerName varchar,summary varchar,unixReviewTime INTEGER,helpful INT,total INT)""")
        filename.encode('utf-8')
        print "Amazon Reviews table executed"
        with open(filename) as f:
            reader = csv.reader(f)
            for field in reader:
                cur.execute("INSERT INTO amazonReviews VALUES (?,?,?,?,?,?,?,?,?,?);", field)

        print "CSV Loaded into SQLite"
        conn.commit()
        conn.close()

c = csvrd().csvFile()<jupyter_output>Amazon Reviews table executed
CSV Loaded into SQLite<jupyter_text>Reading the data from SQLite<jupyter_code>con = sqlite3.connect('amazonReviews.db')

pd.read_sql_query("SELECT * FROM amazonReviews LIMIT 5", con)<jupyter_output><empty_output><jupyter_text>ignoring the reviews having overall score equal to 3. If the overall score is above 3, then the label for it will be set to Positiveelse it will be set to Negative<jupyter_code>reviews = pd.read_sql_query("""
SELECT 
  overall, 
  summary, 
  helpful, 
  total
FROM amazonReviews 
WHERE overall != 3""", con)
#display data
reviews.head(5)<jupyter_output><empty_output><jupyter_text>sentiment column depicts the numeric score of being positive or negativeusefulScore column depicts the boolean value of total number of votes<jupyter_code>reviews["sentiment"] = reviews["overall"].apply(lambda score: "positive" if score > 3 else "negative")
reviews["usefulScore"] = (reviews["helpful"]/reviews["total"]).apply(lambda n: "useful" if n > 0.8 else "useless")

reviews.head(5)<jupyter_output><empty_output><jupyter_text>Rows having overall score equal to 5<jupyter_code>reviews[reviews.overall == 5].head(5)
#Rows having overall score equal to 1
reviews[reviews.overall == 1].head(5)<jupyter_output><empty_output><jupyter_text>Feature extraction from reviewsText Assign new dimension to each word and give the word counts<jupyter_code>regEx = re.compile('[^a-z]+')
def cleanReviews(reviewText):
    reviewText = reviewText.lower()
    reviewText = regEx.sub(' ', reviewText).strip()
    return reviewText

reviews["summaryClean"] = reviews["summary"].apply(cleanReviews)

train, test = train_test_split(reviews, test_size=0.2)
print("%d items in training data, %d in test data" % (len(train), len(test)))<jupyter_output>1018814 items in training data, 254704 in test data<jupyter_text>Using TfidfTransformer().fit_transofrm to fit the train and test data<jupyter_code>countVector = CountVectorizer(min_df = 1, ngram_range = (1, 4))
X_train_counts = countVector.fit_transform(train["summaryClean"])

#applying tfidf to term frequency
tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)

X_new_counts = countVector.transform(test["summaryClean"])
X_test_tfidf = tfidf_transformer.transform(X_new_counts)

y_train = train["sentiment"]
y_test = test["sentiment"]

prediction = dict()
stopwords = set(STOPWORDS)

#mpl.rcParams['figure.figsize']=(8.0,6.0)    #(6.0,4.0)
mpl.rcParams['font.size']=12                #10 
mpl.rcParams['savefig.dpi']=100             #72 
mpl.rcParams['figure.subplot.bottom']=.1 


def show_wordcloud(data, title = None):
    wordcloud = WordCloud(
        background_color='white',
        stopwords=stopwords,
        max_words=200,
        max_font_size=40, 
        scale=3,
        random_state=1 # chosen at random by flipping a coin; it was heads
    ).generate(str(data))
    
    fig = plt.figure(1, figsize=(8, 8))
    plt.axis('off')
    if title: 
        fig.suptitle(title, fontsize=20)
        fig.subplots_adjust(top=2.3)

    plt.imshow(wordcloud)
    plt.show()
    
show_wordcloud(reviews["summaryClean"])
show_wordcloud(reviews[reviews.overall == 1]["summaryClean"], title = "Low scored words")
show_wordcloud(reviews[reviews.overall == 5]["summaryClean"], title = "High scored words")
show_wordcloud(reviews[reviews.overall == 2]["summaryClean"], title = "Average scored words")<jupyter_output><empty_output><jupyter_text>Multinomial Naïve Bayes learning method<jupyter_code>model = MultinomialNB().fit(X_train_tfidf, y_train)
prediction['Multinomial'] = model.predict(X_test_tfidf)<jupyter_output><empty_output><jupyter_text>Bernoulli Naïve Bayes learning method<jupyter_code>model = BernoulliNB().fit(X_train_tfidf, y_train)
prediction['Bernoulli'] = model.predict(X_test_tfidf)<jupyter_output><empty_output><jupyter_text>Logistic regression learning method<jupyter_code>logreg = LogisticRegression(C=1e5)
logreg_result = logreg.fit(X_train_tfidf, y_train)
prediction['Logistic'] = logreg.predict(X_test_tfidf)<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>def formatt(x):
    if x == 'negative':
        return 0
    return 1
vfunc = np.vectorize(formatt)

cmp = 0
colors = ['b', 'g', 'y', 'm', 'k']
for model, predicted in prediction.items():
    false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test.map(formatt), vfunc(predicted))
    roc_auc = auc(false_positive_rate, true_positive_rate)
    plt.plot(false_positive_rate, true_positive_rate, colors[cmp], label='%s: AUC %0.2f'% (model,roc_auc))
    cmp += 1

plt.title('Classifiers comparaison with ROC')
plt.legend(loc='lower right')
plt.plot([0,1],[0,1],'r--')
plt.xlim([-0.1,1.2])
plt.ylim([-0.1,1.2])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text>Visualize the accuracy, recall and f1-score for Logistic Regression<jupyter_code>print(metrics.classification_report(y_test, prediction['Logistic'], target_names = ["positive", "negative"]))
accuracy_score(y_test, prediction['Logistic'])
def plot_confusion_matrix(matrix, title='Confusion matrix', cmap=plt.cm.Blues, labels=["positive", "negative"]):
    plt.imshow(matrix, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(labels))
    plt.xticks(tick_marks, labels, rotation=45)
    plt.yticks(tick_marks, labels)
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    
# Compute confusion matrix
matrix = confusion_matrix(y_test, prediction['Logistic'])
np.set_printoptions(precision=2)
plt.figure()
plot_confusion_matrix(matrix)    

matrix_normalized = matrix.astype('float') / matrix.sum(axis=1)[:, np.newaxis]
plt.figure()
plot_confusion_matrix(matrix_normalized, title='Normalized confusion matrix')
plt.show()<jupyter_output><empty_output><jupyter_text>Visualize the accuracy, recall and f1-score for Naive Bayes Berrnoulli<jupyter_code>print(metrics.classification_report(y_test, prediction['Bernoulli'], target_names = ["positive", "negative"]))
accuracy_score(y_test, prediction['Bernoulli'])
def plot_confusion_matrix(matrix, title='Confusion matrix', cmap=plt.cm.Blues, labels=["positive", "negative"]):
    plt.imshow(matrix, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(labels))
    plt.xticks(tick_marks, labels, rotation=45)
    plt.yticks(tick_marks, labels)
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    
# Compute confusion matrix
matrix = confusion_matrix(y_test, prediction['Bernoulli'])
np.set_printoptions(precision=2)
plt.figure()
plot_confusion_matrix(matrix)    

matrix_normalized = matrix.astype('float') / matrix.sum(axis=1)[:, np.newaxis]
plt.figure()
plot_confusion_matrix(matrix_normalized, title='Normalized confusion matrix')
plt.show()<jupyter_output><empty_output><jupyter_text>Visualize the accuracy, recall and f1-score for Naive Bayes Multinomial<jupyter_code>print(metrics.classification_report(y_test, prediction['Multinomial'], target_names = ["positive", "negative"]))
accuracy_score(y_test, prediction['Multinomial'])
def plot_confusion_matrix(matrix, title='Confusion matrix', cmap=plt.cm.Blues, labels=["positive", "negative"]):
    plt.imshow(matrix, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(labels))
    plt.xticks(tick_marks, labels, rotation=45)
    plt.yticks(tick_marks, labels)
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    
# Compute confusion matrix
matrix = confusion_matrix(y_test, prediction['Multinomial'])
np.set_printoptions(precision=2)
plt.figure()
plot_confusion_matrix(matrix)    

matrix_normalized = matrix.astype('float') / matrix.sum(axis=1)[:, np.newaxis]
plt.figure()
plot_confusion_matrix(matrix_normalized, title='Normalized confusion matrix')
plt.show()<jupyter_output><empty_output><jupyter_text>Displaying words depicting the best and worst featuresusing Logistic Model since it gave the best results<jupyter_code>features = countVector.get_feature_names()
feature_coefs = pd.DataFrame(
    data = list(zip(features, logreg_result.coef_[0])),
    columns = ['feature', 'coefficient'])

feature_coefs.sort_values(by='coefficient')<jupyter_output><empty_output><jupyter_text>Testing the sentiments of few reviews<jupyter_code>def testSentiments(model, testData):
    testCounts = countVector.transform([testData])
    testTfidf = tfidf_transformer.transform(testCounts)
    result = model.predict(testTfidf)[0]
    probability = model.predict_proba(testTfidf)[0]
    print("Sample estimated as %s: negative prob %f, positive prob %f" % (result.upper(), probability[0], probability[1]))

testSentiments(logreg, "Heavenly Highway Hymns")
testSentiments(logreg, "Very oily and creamy. Not at all what I expected... ordered this to try to highlight and contour and it just looked awful!!! Plus, took FOREVER to arrive.")
testSentiments(logreg, "Nasty smelling shampoo!.")<jupyter_output>Sample estimated as POSITIVE: negative prob 0.003317, positive prob 0.996683
Sample estimated as NEGATIVE: negative prob 0.998970, positive prob 0.001030
Sample estimated as NEGATIVE: negative prob 0.934181, positive prob 0.065819
{'fraction_non_alphanumeric': 0.12935595694216384, 'fraction_numerical': 0.014504652435686919, 'mean_word_length': 6.146675358539765, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '655461', 'license': 'MIT', 'n_tokens_mistral': 3864, 'n_tokens_neox': 3656, 'n_words': 744, 'path': 'Amazon reviews_Naive Bayes.ipynb', 'repo_name': 'Samsay143/recommenderSystem-FYP', 'size': '10962'}