<jupyter_start><jupyter_text>Recommender System for E-commerce Use Case<jupyter_code>!pip install h2o4gpu
#Import required libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import math
import h2o4gpu
import itertools
import pickle

from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import LabelEncoder
from math import sqrt
from scipy import sparse
#Reading data
review_data=pd.read_csv('../input/amazon-reviews-dataset/review_data.csv')
#metadata=pd.read_csv('./Data/meta_data.csv')
#Dropping unwanted column
review_data=review_data.drop(['Unnamed: 0'],axis=1)
#metadata=metadata.drop(['Unnamed: 0'],axis=1)<jupyter_output><empty_output><jupyter_text>We have two dataframes-review_data and metadata. review_data is information about the product reviews posted by each user. It has following columns:    reviewerID - ID of the reviewer    asin - ID of the product    reviewerName - name of the reviewer    vote - helpful votes of the review    style - a dictionary of the product metadata    reviewText - text of the review    overall - rating of the product    summary - summary of the review    unixReviewTime - time of the review (unix time)    reviewTime - time of the review (raw)    image - images that users post after they have received the productmetadata includes information about the product:    asin - ID of the product    title - name of the product    feature - bullet-point format features of the product    description - description of the product    price - price in US dollars (at time of crawl)    image - url of the product image    related - related products (also bought, also viewed, bought together, buy after viewing)    salesRank - sales rank information    brand - brand name    categories - list of categories the product belongs to    tech1 - the first technical detail table of the product    tech2 - the second technical detail table of the product    similar - similar product table    Citation: Justifying recommendations using distantly-labeled reviews and fined-grained aspects<NAME>, <NAME>, <NAME>yEmpirical Methods in Natural Language Processing (EMNLP), 2019pdf<jupyter_code>review_data.head()
#metadata.head()<jupyter_output><empty_output><jupyter_text>Taking a subset of the large review dataset as per our requirement<jupyter_code>sub_matrix = review_data[['reviewerID', 'asin', 'overall']].copy().drop_duplicates()
#sub_matrix.set_index('reviewerID')
sub_matrix.shape
#Saving user data into pickle files
u=np.unique(sub_matrix.reviewerID)
user_dict = { k : [] for k in u }
for row in sub_matrix.iterrows():
    new_lst=(row[1]['asin'],row[1]['overall'])
    user_dict[row[1]['reviewerID']].append(new_lst)<jupyter_output><empty_output><jupyter_text>Pre-processing<jupyter_code>#Taking a sub-set of data for training
train=sub_matrix[0:500000]<jupyter_output><empty_output><jupyter_text>We have to filter the test set to have rows for only those products and users present in the training data.<jupyter_code>#Preparing the test set
test=sub_matrix[500000:]
#test=remaining[sub_matrix['reviewerID'].isin(user_train) & sub_matrix['asin'].isin(item_train)]
#test=test[test.reviewerID.isin(train.reviewerID) & test.asin.isin(train.asin)]
test = test[test.reviewerID.isin(train.reviewerID)]
test = test[test.asin.isin(train.asin)]
user_le = LabelEncoder()
item_le = LabelEncoder()

train.reviewerID = user_le.fit_transform(train.reviewerID)
test.reviewerID = user_le.transform(test.reviewerID)

train.asin = item_le.fit_transform(train.asin)
test.asin = item_le.transform(test.asin)
print(train.shape)
print(test.shape)
user_train, user_train_indices = np.unique(np.array(train['reviewerID'], dtype="int32"), return_inverse=True)
item_train, item_train_indices = np.unique(np.array(train['asin'], dtype="int32"), return_inverse=True)
#Now for test set
user_test,user_test_indices= np.unique(np.array(test['reviewerID'], dtype="int32"), return_inverse=True)
item_test,item_test_indices=np.unique(np.array(test['asin'], dtype="int32"),return_inverse=True)
test=test.reset_index(drop=True)
print(len(user_train))
print(len(item_train))
print(len(user_test))
print(len(item_test))<jupyter_output>444449
47580
9820
5639<jupyter_text>Creating and computing error values for simple baseline models<jupyter_code>pred_global_avg = [train.overall.mean()] * test.shape[0]
product_avg_preds = train.groupby('asin')['overall'].mean().reset_index().rename(
    columns={'overall': 'pred_product_avg'}
)
user_avg_preds = train.groupby('reviewerID')['overall'].mean().reset_index().rename(
    columns={'overall': 'pred_user_avg'}
)
#Merging with test
test = test.merge(product_avg_preds, on='asin')
test = test.merge(user_avg_preds, on='reviewerID')
test['pred_global_avg'] = pred_global_avg
sparse_train=sparse.coo_matrix((np.array(train.overall.values, dtype='float32'),(user_train_indices,item_train_indices)),shape=(len(user_train), len(item_train)))
sparse_test=sparse.coo_matrix((np.ones(len(test), dtype="float32"),(user_test_indices,item_test_indices)),shape=(sparse_train.shape))
print(sparse_train.shape)
print(sparse_test.shape)<jupyter_output>(444449, 47580)
(444449, 47580)<jupyter_text>Using h2o4GPU  Hyperparameter Tuning<jupyter_code>components=[10, 20, 30, 50, 70]
lambda_values=[0.01, 0.05, 0.1, 0.5]
max_iterations=[50, 100, 200]
parameter_values=list(itertools.product(components,lambda_values,max_iterations))
#Factorization function
def execute_mf(parameters,X,X_test,model=False):
    factorization = h2o4gpu.solvers.FactorizationH2O(parameters[0], parameters[1], parameters[2])
    factorization.fit(X, X_BATCHES=10, THETA_BATCHES=10)
    predictions=factorization.predict(X_test).data
    if model:
        return predictions,factorization
    else:
        return predictions
results=dict()
c=0
for i in parameter_values:#parameter_values[0:math.floor(len(parameter_values)/2)]:
    c=c+1
    preds=execute_mf(i,sparse_train,sparse_test)
    if not np.isnan(preds).any():
        results[i]=np.sqrt(mean_squared_error(test.overall, preds))
    if c%5==0:
        print('Processing done. combinations left : ',len(parameter_values)-c)
#Fetching the parameter combination giving least RMSE value
print(min(results, key=results.get))<jupyter_output>(70, 0.01, 200)<jupyter_text>Thus, we infer that (70, 0.01, 200) is the best combination of parameter values. Hence we run our algorithm using that.<jupyter_code>#pred_mf,model=execute_mf([70,0.01,200],sparse_train,sparse_test,True)
print(f'RMSE Product Avg: {np.sqrt(mean_squared_error(test.overall, test.pred_product_avg))}')
print(f'RMSE User Avg: {np.sqrt(mean_squared_error(test.overall, test.pred_user_avg))}')
print(f'RMSE Global Avg: {np.sqrt(mean_squared_error(test.overall, test.pred_global_avg))}')
print(f'RMSE H2O-MF: ',results[(70, 0.01, 200)])#np.sqrt(mean_squared_error(test.overall, test.pred_mf))}')<jupyter_output>RMSE Product Avg: 1.4675429714630877
RMSE User Avg: 1.6234457217469518
RMSE Global Avg: 1.4346134925714396
RMSE H2O-MF:  2.0201818047054627<jupyter_text>Saving our final model alongwith required data<jupyter_code>user_encode = LabelEncoder()
item_encode = LabelEncoder()

sub_matrix.reviewerID = user_encode.fit_transform(sub_matrix.reviewerID)
sub_matrix.asin = item_encode.fit_transform(sub_matrix.asin)

user, user_indices = np.unique(np.array(sub_matrix['reviewerID'], dtype="int32"), return_inverse=True)
item, item_indices = np.unique(np.array(sub_matrix['asin'], dtype="int32"), return_inverse=True)

sparse_mat=sparse.coo_matrix((np.array(sub_matrix.overall.values, dtype='float32'),(user_indices,item_indices)),shape=(len(user), len(item)))
i=item_encode.inverse_transform(item)
#Preparing Item-Index mapping dictionary
items=dict(zip(i,item))
u=user_encode.inverse_transform(user)
#Preparing User-Index mapping dictionary
users=dict(zip(u,user))
print(len(users))
print(len(items))
print(sparse_mat.shape)
rec = h2o4gpu.solvers.FactorizationH2O(70, 0.01, 200)
rec.fit(sparse_mat, X_BATCHES=10, THETA_BATCHES=10)
#Saving user data into pickle files
#items=item_encode.inverse_transform(item)
d_dict={'user':user_dict,'user_indices':users,'item_indices':items}
with open('review_data.pickle','wb') as f:
    pickle.dump(d_dict,f,pickle.HIGHEST_PROTOCOL)
#Extracting User's features and Item's features from model
#user_matrix=rec.XT
#item_matrix=rec.thetaT
#Getting users and items
#users=user_encode.inverse_transform(user)
#items=item_encode.inverse_transform(item)
#users=list(map(str,users))
#items=list(map(str,items))
#Serializing predictions and users and items
#recommend={'user_matrix':user_matrix,'item_matrix':item_matrix,'users':users,'items':items}
#with open('recommend_amazon.pickle', 'wb') as f:
#    # Pickle the 'data' dictionary using the highest protocol available.
#    pickle.dump(recommend, f, pickle.HIGHEST_PROTOCOL)
#Saving model file as pickle
with open('recommend_model.pickle','wb') as f:
    pickle.dump(rec,f,pickle.HIGHEST_PROTOCOL)
#Create dictionary of mini dataframes
#user_data=dict()
#user_data={row[1]['reviewerID']:sub_matrix[sub_matrix['reviewerID']==row[1]['reviewerID']] for row in sub_matrix.iterrows()}
#user_data={key['reviewerID']:pd.DataFrame.from_records([l for l in df_dict if l['reviewerID']==key['reviewerID']]) for key in df_dict}

#u=user_encode.inverse_transform(user)
#user_data={val:sub_matrix[sub_matrix['reviewerID']==val][['asin','overall']] for val in u}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12453630100688924, 'fraction_numerical': 0.02702702702702703, 'mean_word_length': 7.395017793594306, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '451410', 'license': 'Apache-2.0', 'n_tokens_mistral': 3382, 'n_tokens_neox': 3183, 'n_words': 672, 'path': 'recommendation-system-using-matrix-factorization.ipynb', 'repo_name': 'shrinidhin/recommender-system', 'size': '9450'}