<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive/')
cd 'drive/My Drive/Colab Datasets'
ls
!pip install -q imgaug==0.2.6
!pip install -q matplotlib==3.1.0  # 3.1.1 issues with seaborn heatmap.
# Import data science environment.
import os
import pickle
import re

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer

# Display preferences.
%matplotlib inline
pd.options.display.float_format = '{:.3f}'.format
sns.set_style('white')
print('pandas', pd.__version__)
print('numpy', np.__version__)
print('seaborn', sns.__version__)
UK_df = pd.read_csv('./amazon_reviews_UK.tsv', sep='\t', error_bad_lines=False,
                    parse_dates=['review_date'])
US_df = pd.read_csv('./amazon_reviews_US.tsv', sep='\t', error_bad_lines=False,
                    parse_dates=['review_date'])
UK_df.shape
US_df.shape
# Count lines in df.
cat amazon_reviews_UK.tsv | wc -l
cat amazon_reviews_US.tsv | wc -l
UK_df.head()
assert US_df.shape[1] == UK_df.shape[1]
assert UK_df.columns.all() == US_df.columns.all()
combined_df = pd.concat([UK_df.copy(), US_df.copy()])
assert combined_df.shape[0] == US_df.shape[0] + UK_df.shape[0]
combined_df.reset_index(inplace=True)
combined_df.head()
combined_df.isnull().sum()
combined_df = combined_df.drop_duplicates()
# Drop rows with null values.
combined_df = combined_df.dropna()
combined_df.isnull().sum()
combined_df.dtypes
# Change dtypes from float64 to int64.
combined_df.star_rating = combined_df.star_rating.astype(int)
combined_df.helpful_votes = combined_df.helpful_votes.astype(int)
combined_df.total_votes = combined_df.total_votes.astype(int)
combined_df.dtypes
combined_df.review_body.head()
cols_to_drop = ['customer_id', 'review_id', 'product_id', 'product_parent',
                'product_title', 'review_headline', 'review_date']
                
combined_df = combined_df.drop(cols_to_drop, axis=1)
combined_df['helpful_rate'] = np.where(
  combined_df['total_votes'] > 0,
  combined_df['helpful_votes'] / combined_df['total_votes'], -1)
combined_df.info()
# Change objects to category types where appropriate.
combined_df['marketplace'] = combined_df.marketplace.astype('category')
combined_df['vine'] = combined_df.vine.astype('category')
combined_df['verified_purchase'] = combined_df.verified_purchase.astype(
    'category')
combined_df['product_category'] = combined_df.product_category.astype(
    'category')
combined_df.info()
# Split into train and eval/holdout groups.
X = combined_df.drop('star_rating', axis=1)
y = combined_df.star_rating
X_train, X_eval_holdout, y_train, y_eval_holdout = train_test_split(
    X, y, test_size=0.3, random_state=15)
# Split into eval and holdout groups.
X_eval, X_holdout, y_eval, y_holdout = train_test_split(
    X_eval_holdout, y_eval_holdout, test_size=0.5, random_state=15)
X_train.nunique()
# Run heatmap to check for variable correlations.
fig, ax = plt.subplots(figsize=(12,10))
sns.heatmap(X_train.corr(), annot=True, annot_kws={"size": 10})
plt.tight_layout();<jupyter_output><empty_output><jupyter_text>It appears that total_votes and helpful_votes are highly correlated for both data sets. Therefore, we should drop one of them. Not sure which one to drop at this point, but will need to discuss it and decide.<jupyter_code>X_train.vine.value_counts()
y_train.value_counts().sort_index(ascending=False)
sns.countplot(y_train).set_title(
    "Rating Totals for UK and US")
plt.xlabel('Star Rating')
plt.ylabel('Count');
# Pie chart
agg = y_train.value_counts().sort_index(ascending=False)
labels = agg.index
sizes = agg.values
# only "explode" the largest slice
explode = (0.1, 0, 0, 0, 0)  
fig1, ax1 = plt.subplots(figsize=(8, 8))
ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',
        shadow=True, startangle=90) 
plt.tight_layout()
plt.title('Review Star Rating Distribution')
plt.show()
uk = X_train.loc[X_train.marketplace == 'UK', 'product_category'].value_counts()
us = X_train.loc[X_train.marketplace == 'US', 'product_category'].value_counts()

print(uk)
print()
print(us)
uk_length = (X_train.marketplace == 'UK').sum()
us_length = (X_train.marketplace == 'US').sum()

uk_percent = (uk / uk_length).sort_index()
us_percent = (us / us_length).sort_index()
fig, ax = plt.subplots(1, 2, figsize=(15, 10), sharey=True)

ax[0].bar(uk_percent.index, uk_percent.values, color='r')
ax[0].set_title('UK Product Category Percents')
ax[0].set_xticklabels(uk_percent.index, rotation='vertical')
ax[0].set_ylabel('Purchase Percent')
ax[0].set_xlabel('Purchase Category')

ax[1].bar(us_percent.index, us_percent.values, color='b')
ax[1].set_title('US Product Category Percents')
ax[1].set_xticklabels(us_percent.index, rotation='vertical')
ax[1].set_xlabel('Purchase Category')

plt.tight_layout()
plt.show();
pd.concat(
  [X_train, y_train], axis=1).groupby(
    'marketplace')['star_rating'].agg(['mean', 'std'], axis=1)
pd.concat(
  [X_train, y_train], axis=1).groupby(
    ['product_category','marketplace'])['star_rating'].agg(
      ['mean', 'std'], axis=1)
_ = pd.concat(
  [X_train, y_train], axis=1).groupby(
    ['product_category', 'marketplace'])['star_rating'].mean()
_ = _.reset_index() # convert from multi-index series to df.

plt.figure(figsize=(15, 8))
plt.title('Star Ratings by Product Category and Marketplace')
sns.barplot(x=_.product_category, y=_.star_rating, hue=_.marketplace,)
plt.xlabel('Product Category')
plt.ylabel('Star Rating')
plt.xticks(rotation='vertical');
_ = pd.concat([X_train, y_train], axis=1)
_ = _[_.helpful_rate != -1]
_.groupby(['marketplace', 'star_rating'])['helpful_rate'].mean()
_ = _.reset_index()

plt.figure(figsize=(15, 8))
plt.title('Helpful Ratings by Star Ratings and Marketplace')
sns.barplot(x=_.marketplace, y=_.helpful_rate, hue=_.star_rating)
plt.xlabel('Marketplace')
plt.ylabel('Helpful Rate')
plt.xticks(rotation='vertical');
plt.figure(figsize=(15, 8))
plt.xticks(rotation='vertical')
sns.barplot(x='product_category',
            y='helpful_votes',
            hue='marketplace',
            data=X_train, ci=None)
plt.legend(labels=['UK', 'US'])
plt.xlabel('Product Category')
plt.ylabel('Helpful Votes')
plt.title('Helpful Votes by Product Category and Marketplace');<jupyter_output><empty_output><jupyter_text>Why is UK Lawn and Garden such an extreme outlier? After looking into the data, it appears that only 186 of the reviews were from British Lawn & Garden customers. Of those, approximately eleven were helpful reviews, which yields such an extremely higher percentage. With more reviews in that category, there is great likelihood that the helpful vote percentage would even out.<jupyter_code>X_train_L_and_G_count = X_train[(X_train.product_category == 'Lawn and Garden') &
                                (X_train.marketplace == 'UK')].shape[0]

X_train_L_and_G_positive_review_count = X_train[
  (X_train.product_category == 'Lawn and Garden') &
  (X_train.marketplace == 'UK') &
  (X_train.helpful_votes == 1)].shape[0]

print('Number of Lawn and Garden Reviews from the UK:', X_train_L_and_G_count)
print('Helpful Reviews for UK Lawn & Garden:',
    X_train_L_and_G_positive_review_count)
print('Percent Helpful Review:', round(
    (X_train_L_and_G_positive_review_count * 100 / X_train_L_and_G_count), 3))
X_train.groupby(['product_category', 'marketplace'])['helpful_votes'].agg(
    ['mean', 'std'])
# Group by Verified users only.
X_train.groupby(
    ['verified_purchase', 'marketplace'])['helpful_rate'].agg(['mean', 'std'])
plt.figure(figsize=(15, 8))
sns.barplot(x='verified_purchase',
            y='helpful_votes',
            hue='marketplace',
            data=X_train, ci=None)
plt.legend(['UK', 'US'])
plt.xticks(labels=['Non-verified purchase', 'Verified purchase'], ticks=[0, 1])
plt.title('Helpful Votes by Verified Purchase and Marketplace')
plt.xlabel('')
plt.ylabel('Helpful Votes');
# Group by Vine users only.
X_train.groupby(
    ['vine', 'marketplace'])['helpful_rate'].agg(['mean', 'std'])
plt.figure(figsize=(15, 8))
sns.barplot(x='vine',
            y='helpful_votes',
            hue='marketplace',
            data=X_train, ci=None)
plt.legend(['UK', 'US'])
plt.xticks(ticks=[0, 1], labels=['Non-Vine Review', 'Vine Review'])
plt.title('Mean Helpful Vote Tags by Vine Reviewers and Marketplace')
plt.xlabel('Vine Membership')
plt.ylabel('Mean Helpful Vote Tag');
# Distribution of vine transactions.
X_train.vine.value_counts()
_ = pd.concat([X_train, y_train], axis=1)
_.head()

l_and_g = _[_['product_category'] == 'Lawn and Garden']
l_and_g.groupby('marketplace')['helpful_votes'].value_counts().sort_index()
X_train[X_train['helpful_rate'] > 0]['helpful_rate'].hist(bins=40)
plt.xlabel('Helpful Percent')
plt.ylabel('Review Count')
plt.title('Helpful Percentage Counts with Reviews Present');<jupyter_output><empty_output><jupyter_text>Upsample/downsample next? Use all UK data and upsample to US data, or upsample some and downsample some? If the latter, should the new data be jittered (upsampling only)?<jupyter_code>X_train.describe()
X_train.helpful_votes.mean()
X_train.loc[X_train.helpful_votes > 0, 'helpful_votes'].std()
X_train.describe()
def shrink_outliers(feature: str, train_df: pd.DataFrame, eval_df: pd.DataFrame,
                    holdout_df: pd.DataFrame):  
  """
  Takes a feature and replaces values which are greater than three standard
  deviations from the mean with the value at that point. 
  
  Args:
    feature: the column with outliers
    train_df: DataFrame from which mean and standard deviation will be 
      calculated.
    eval_df: DataFrame will have mean and standard deviation applied to feature
      column.
    holdout_df: DataFrame will have mean and standard deviation applied to
      feature column.
  Returns:
    train_df: DataFrame with outliers replaced.
    eval_df: DataFrame with outliers replaced.
    holdout_df: DataFrame with outliers replaced.
  """

  mean = train_df.loc[train_df[feature] != 0, feature].mean()
  std = train_df.loc[train_df[feature] != 0, feature].std()
  replacement_value = mean + (3 * std)

  train_df[feature] = train_df[feature].apply(
      lambda x: x if x < replacement_value else replacement_value).copy()
  eval_df[feature] = eval_df[feature].apply(
      lambda x: x if x < replacement_value else replacement_value).copy()
  holdout_df[feature] = holdout_df[feature].apply(
      lambda x: x if x < replacement_value else replacement_value).copy() 
  
  return train_df, eval_df, holdout_df
X_train = X_train.copy()
X_eval = X_eval.copy()
X_holdout = X_holdout.copy()
# Add column with length of review_body.
X_eval['review_body_len'] = X_eval.review_body.str.split().apply(len)
X_holdout['review_body_len'] = X_holdout.review_body.str.split().apply(len)
# Ran into memory issues using vectorized methods. Switched to for loop to 
# calculate lengths.
lengths = []
for i, text in X_train.review_body.iteritems():
  lengths.append(len(text.split()))

assert X_train.shape[0] == len(lengths)
X_train['review_body_len'] = lengths
# pd.set_option('display.max_columns', None)  
# pd.set_option('display.expand_frame_repr', False)
# pd.set_option('max_colwidth', -1)
X_train.loc[:, ['review_body', 'review_body_len']].head()
X_train.describe()
X_train.review_body_len.hist(bins=200)
plt.xlabel('Raw Review Length')
plt.ylabel('Count')
plt.title('Distribution of Review Lengths for X_train');
X_train, X_eval, X_holdout = shrink_outliers(
    'review_body_len', X_train, X_eval, X_holdout)
X_train.describe()
X_train.review_body_len.hist(bins=200)
plt.xlabel('Review Length Smoothed')
plt.ylabel('Count')
plt.title('Distribution of Review Lengths for X_train');
# Text cleaning.
def _strip_html(text):
    soup = BeautifulSoup(text, "html.parser")
    return soup.get_text()


def _remove_between_square_brackets(text):
    return re.sub('\[[^]]*\]', '', text)


def denoise_text(text):
    text = _strip_html(text)
    text = _remove_between_square_brackets(text)
    return text
X_train['review_body'] = X_train.review_body.apply(denoise_text)
X_eval['review_body'] = X_eval.review_body.apply(denoise_text)
X_holdout['review_body'] = X_holdout.review_body.apply(denoise_text)
os.getcwd()
vect = CountVectorizer(encoding='utf-8', lowercase=True, stop_words='english',
                       ngram_range=(1,1), min_df=10, max_features=5_000,
                       dtype=np.int32)
X_train_text_vect = vect.fit_transform(X_train.review_body)
X_eval_text_vect = vect.transform(X_eval.review_body)
X_holdout_text_vect = vect.transform(X_holdout.review_body)
type(X_eval_text_vect)
scipy.sparse.save_npz('X_train_text_sparse', X_train_text_vect)
scipy.sparse.save_npz('X_eval_text_sparse', X_eval_text_vect)
scipy.sparse.save_npz('X_holdout_text_sparse', X_holdout_text_vect)
X_train['target'] = y_train
X_eval['target'] = y_eval
X_holdout['target'] = y_holdout
X_train.to_csv('train_clean.tsv', index=False, sep='\t')
X_eval.to_csv('eval_clean.tsv', index=False, sep='\t')
X_holdout.to_csv('holdout_clean.tsv', index=False, sep='\t')
text_col_names = vect.get_feature_names()
len(text_col_names)
with open('text_col_names.csv', 'a') as f:
  for col in text_col_names:
    f.write('{}, '.format(col))
ls -lah<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1400702593616862, 'fraction_numerical': 0.009716720233201286, 'mean_word_length': 5.630327056491576, 'pattern_counts': {'":': 1, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '406348', 'license': 'MIT', 'n_tokens_mistral': 4907, 'n_tokens_neox': 4731, 'n_words': 989, 'path': 'final_capstone_00_eda.ipynb', 'repo_name': 'serfsup/thinkful-final-capstone', 'size': '13379'}