<jupyter_start><jupyter_text><jupyter_code>#@title Default title text
#download file 
!wget https://s3.amazonaws.com/amazon-reviews-pds/tsv/amazon_reviews_us_Mobile_Electronics_v1_00.tsv.gz



import gzip
import shutil
with gzip.open('amazon_reviews_us_Mobile_Electronics_v1_00.tsv.gz', 'rb') as f_in:
    with open('amazon_reviews_us_Mobile_Electronics_v1_00.tsv', 'wb') as f_out:
        shutil.copyfileobj(f_in, f_out)
#read amazon reviews dataset
import pandas as pd
df = pd.read_csv("amazon_reviews_us_Mobile_Electronics_v1_00.tsv",sep="\t", error_bad_lines=False)
df_full = df
df.head()

#download sentiment analysis
!wget https://www2.imm.dtu.dk/pubdb/views/edoc_download.php/6010/zip/imm6010.zip

import zipfile
with zipfile.ZipFile("imm6010.zip", 'r') as zip_ref:
    zip_ref.extractall()
#read AFINN-111
afinn = pd.read_csv("AFINN/AFINN-111.txt",sep="\t", error_bad_lines=False, header = None, names = ["word", "rate"])
afinn.head()

#cast afinn to dictionary
afinn_dict = dict(zip(afinn.word, afinn.rate))

#core of the script: sentiment analysis
import numpy as np


#add two new columns to amazon dataframe: number of matched words with afinn dataset and sum of afinn ratings for matched words 
df["matched"] = np.nan
df["matched_rate"] = np.nan

for index, row in df.iterrows():
    #create tokenized words
    try:
      #split review body into single tokens
      tokens = row["review_body"].lower().replace('.', '').split(' ')
      counter = 0
      afinn_rate = 0
      #print("processing record nr : {0}".format(index))
      for elem in tokens :      
        try :
          #look for review word in afinn dataset
          afinn_rate += afinn_dict[elem]
          #if there is a match, increase counter by one
          counter += 1
        except KeyError: 
          l = 0
      #update values for that record 
      df.at[index,'matched'] = counter
      df.at[index,'matched_rate'] = afinn_rate
    
    except AttributeError:
      m = 0
    #print(tokens)
#    if index == 10 :
#      break

df.head()
###TEST FOR CORRELATION LINE
import matplotlib.pyplot as plt
from google.colab import files
from scipy import stats
#compute sentiment analysis rating as  matched_rate / number of matched words
df["avg_matched"] = df["matched_rate"] / df["matched"]

df = df.dropna()

#rescale afinn rating in range 1-5 (originally it was between -4 and +4)
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler((1, 5))
df["avg_matched"] = scaler.fit_transform(df[["avg_matched"]])

#plot results
#scatter = df.plot(x='avg_matched', y='star_rating', style='o', grid = True, legend=False, figsize=(12,10))
x=df['avg_matched']
y=df['star_rating']
m, b = np.polyfit(x, y, 1)
plt.figure(figsize=(12,10))
plt.scatter(x, y) 
plt.title('Correlation between star rating and reviews sentiment analysis\n', fontsize=20)
plt.xlabel('Sentiment Analysis Rating', fontsize=16)
plt.ylabel('Amazon Star Rating', fontsize=16)
plt.grid()

cor = stats.pearsonr(df["star_rating"], df["avg_matched"])[0]
plt.plot(np.unique(x), np.poly1d(np.polyfit(x, y, cor))(np.unique(x)), color='orange')
plt.show()

#scatter.set_xlabel("Sentiment Analysis Rating")
#scatter.set_ylabel("Amazon Star Rating")


#compute pearson correlation
print("\n\tPearson correlation: {0}".format(stats.pearsonr(df["star_rating"], df["avg_matched"])[0]))

df['difference'] = df['star_rating'] - df['avg_matched']

ok = df[(df['difference'] <= 1.2) & (df['difference'] >= -1.2)]
good_records = ok.shape[0]
total_records = df_full.shape[0]

print('\trecords acceptable (max Delta ±1): {0} out of {1} ({2}%)'.format(good_records, total_records, round((good_records/total_records), 2)*100))

#uncomment to download a csv file of the result
#df['avg_rounded'] = round(df['avg_matched'])
#df.to_csv('NLP_result.csv', sep='\t' ,header=True, index = False)
#files.download("NLP_result.csv")
test = df_full[(df_full['star_rating'] == 2)]
print('{0} records'.format(len(test)))
test.head()



#df.head()
#df.to_csv('output.csv', index=False, sep = '\t')<jupyter_output>7298 records
{'fraction_non_alphanumeric': 0.1488521352752407, 'fraction_numerical': 0.020982473463342383, 'mean_word_length': 4.71509167842031, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '25507', 'license': 'CC0-1.0', 'n_tokens_mistral': 1497, 'n_tokens_neox': 1424, 'n_words': 346, 'path': 'BDA_CW2.ipynb', 'repo_name': 'bianchin90/Projects', 'size': '4051'}