<jupyter_start><jupyter_text>1. Packages Import<jupyter_code>import pandas as pd
import numpy as np
import os
import json
import tempfile
import datetime
import re, string, unicodedata
import nltk
from nltk import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk import SnowballStemmer
from nltk.stem import LancasterStemmer, WordNetLemmatizer
import matplotlib.pyplot as pl

import pickle
import matplotlib.pyplot as plt
import itertools
from functools import partial
from itertools import product

from sklearn import metrics
from sklearn.model_selection import train_test_split
from keras.preprocessing.sequence import pad_sequences
from sklearn.metrics import confusion_matrix, classification_report

import tensorflow as tf
from tensorflow import keras
from keras import backend as K
from keras.preprocessing.text import Tokenizer
from keras.models import Sequential, Model
from keras.models import model_from_json
from keras.layers import Dense, Input, Flatten, Embedding, Conv1D, SpatialDropout1D, MaxPooling1D, Dropout, Reshape, LSTM, CuDNNLSTM
from keras.layers import GlobalAveragePooling1D
from keras.layers.merge import Concatenate
from keras.utils import to_categorical
from keras.optimizers import Adam, RMSprop
from keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping
from keras.models import load_model<jupyter_output>Using TensorFlow backend.<jupyter_text>2. Data Loading- Load and read the data in google cloud - Link is the sharable link in google drive<jupyter_code>!pip install -U -q PyDrive
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials
# Authenticate and create the PyDrive client.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)
def read_csv(uploaded, name):
  from io import StringIO
  s=str(uploaded[name],'utf-8')
  data = StringIO(s) 
  df=pd.read_csv(data)
  return df
## reviews data
link = "https://drive.google.com/open?id=1cXu9-KprFIUmPsFWT1TRR3UTc-6A07MI"
_, drive_id = link.split('=')
downloaded = drive.CreateFile({'id': drive_id})
downloaded.GetContentFile('./sample.csv')
reviews = pd.read_csv('./sample.csv')
print(len(reviews))
reviews.head()
sample = reviews.iloc[:,:3]
sample.columns = ['UserId', 'ItemId', 'Rating']
## business side info data after doing MCA
link = "https://drive.google.com/open?id=1-RfRHys9E9_yM1GgYVkgNhCEzHwZY4yn"
_, drive_id = link.split('=')
downloaded = drive.CreateFile({'id': drive_id})
downloaded.GetContentFile('./mca_360.csv')
mca_360 = pd.read_csv('./mca_360.csv', index_col = 0)
print(len(mca_360))
mca_360.head()
## Complete business side info data
link = "https://drive.google.com/open?id=1UacX5fkE-p3pLC9rdzow-74JIkmXA6Yn"
_, drive_id = link.split('=')
downloaded = drive.CreateFile({'id': drive_id})
downloaded.GetContentFile('./bus_sideinfo.csv')
bus = pd.read_csv('./bus_sideinfo.csv', index_col = 0)
bus.head()
mca_360['ItemId']= bus['business_id']
mca_360.head()

item_side_info = mca_360<jupyter_output><empty_output><jupyter_text>3. Data Processing  3.1 reviews data train-test split<jupyter_code>np.random.seed(1)
user_ids = sample.UserId.drop_duplicates().values
item_ids = sample.ItemId.drop_duplicates().values
users_train = set(np.random.choice(user_ids, size=int(user_ids.shape[0] * .75), replace=False))
items_train = set(np.random.choice(item_ids, size=int(item_ids.shape[0] * .75), replace=False))
train = sample.loc[sample.UserId.isin(users_train) & sample.ItemId.isin(items_train)].reset_index(drop=True)

np.random.seed(1)
train_ix = train.sample(frac=.85).index
test_ix = np.setdiff1d(train.index.values, train_ix)
test_warm_start = train.loc[test_ix].reset_index(drop=True)
train = train.loc[train_ix].reset_index(drop=True)
users_train = set(train.UserId)
items_train = set(train.ItemId)
test_warm_start = test_warm_start.loc[test_warm_start.UserId.isin(users_train) &
                                      test_warm_start.ItemId.isin(items_train)].reset_index(drop=True)

test_cold_start = sample.loc[~sample.UserId.isin(users_train) & ~sample.ItemId.isin(items_train)].reset_index(drop=True)
test_new_items = sample.loc[(sample.UserId.isin(users_train)) & (~sample.ItemId.isin(items_train))].reset_index(drop=True)
test_new_items_set = set(test_new_items.ItemId)

print(train.shape)
print(test_warm_start.shape)
print(test_new_items.shape)<jupyter_output>(551836, 3)
(92936, 3)
(219959, 3)<jupyter_text>3.2 business side info processing<jupyter_code>items_w_sideinfo = set(item_side_info.ItemId)
test_new_items = test_new_items.loc[test_new_items.ItemId.isin(items_w_sideinfo)].reset_index(drop=True) 

item_sideinfo_train = item_side_info.loc[item_side_info.ItemId.isin(items_train)].reset_index(drop=True) 
item_sideinfo_testnew = item_side_info.loc[item_side_info.ItemId.isin(test_new_items_set)].reset_index(drop=True) 
test_cold_start = test_cold_start.loc[test_cold_start.ItemId.isin(items_w_sideinfo)].reset_index(drop=True)
print(len(test_new_items))
item_sideinfo_train.head()<jupyter_output><empty_output><jupyter_text>4. Models  4.1 Basic Collective Matrix Factorization Model - Only ratings<jupyter_code>!pip install cmfrec
from copy import deepcopy
from cmfrec import CMF
%%time
#k=40, warm start
model_no_side_info = CMF(k=40, reg_param=1e-4, random_seed=1)
model_no_side_info.fit(deepcopy(train))
test_warm_start['Predicted'] = model_no_side_info.predict(test_warm_start.UserId, test_warm_start.ItemId)
RMSE_noside = []

RMSE_warm_k40 = np.sqrt(np.mean( (test_warm_start.Predicted - test_warm_start.Rating)**2))
RMSE_noside.append(RMSE_warm_k40)
print("RMSE (no side info, warm start), k=40: ",  RMSE_warm_k40)
%%time
#k=50, warm start
model_no_side_info_50 = CMF(k=50, reg_param=1e-4, random_seed=1)
model_no_side_info_50.fit(deepcopy(train))
test_warm_start['Predicted_50'] = model_no_side_info_50.predict(test_warm_start.UserId, test_warm_start.ItemId)
RMSE_warm_k50 = np.sqrt(np.mean( (test_warm_start.Predicted_50 - test_warm_start.Rating)**2))
RMSE_noside.append(RMSE_warm_k50)
print("RMSE (no side info, warm start), k=50: ",  RMSE_warm_k50)
%%time
#k=35, warm start
model_no_side_info_35 = CMF(k=35, reg_param=1e-4, random_seed=1)
model_no_side_info_35.fit(deepcopy(train))
test_warm_start['Predicted_35'] = model_no_side_info_35.predict(test_warm_start.UserId, test_warm_start.ItemId)
RMSE_warm_k35 = np.sqrt(np.mean( (test_warm_start.Predicted_35 - test_warm_start.Rating)**2))
RMSE_noside.append(RMSE_warm_k35)
print("RMSE (no side info, warm start), k=35: ",  RMSE_warm_k35)
RMSE_noside<jupyter_output><empty_output><jupyter_text>4.2 Model with business side info<jupyter_code>#k=35, all side info
%%time
model_item_info_all = CMF(k=35, k_main=15, k_item=10, reg_param=1e-3, w_main=10.0, w_item=0.5, random_seed=1)
model_item_info_all.fit(deepcopy(train), item_info = deepcopy(item_side_info))
test_warm_start['PredictedAll_35'] = model_item_info_all.predict(test_warm_start.UserId, test_warm_start.ItemId)
test_new_items['PredictedAll_35'] = model_item_info_all.predict(test_new_items.UserId, test_new_items.ItemId)
RMSE_sideall_warm = []
RMSE_sideall_newitem=[]

RMSE_sideall_warm_k35 = np.sqrt(np.mean( (test_warm_start.PredictedAll_35 - test_warm_start.Rating)**2))
RMSE_sideall_warm.append(RMSE_sideall_warm_k35)
print("RMSE (side info, warm start), k=35: ",  RMSE_warm_sideall_k35)

RMSE_sideall_newitem_k35 = np.sqrt(np.mean( (test_new_items.PredictedAll_35 - test_new_items.Rating)**2))
RMSE_sideall_newitem.append(RMSE_sideall_newitem_k35)
print("RMSE (side info, new item), k=35: ",  RMSE_sideall_newitem_k35)
#k=40, all side info
%%time
model_item_info_all = CMF(k=40, k_main=15, k_item=10, reg_param=1e-3, w_main=10.0, w_item=0.5, random_seed=1)
model_item_info_all.fit(deepcopy(train), item_info = deepcopy(item_side_info))
test_warm_start['PredictedAll_40'] = model_item_info_all.predict(test_warm_start.UserId, test_warm_start.ItemId)
test_new_items['PredictedAll_40'] = model_item_info_all.predict(test_new_items.UserId, test_new_items.ItemId)
RMSE_sideall_warm_k40 = np.sqrt(np.mean( (test_warm_start.PredictedAll_40 - test_warm_start.Rating)**2))
RMSE_sideall_warm.append(RMSE_sideall_warm_k40)
print("RMSE (side info, warm start), k=40: ",  RMSE_sideall_warm_k40)
RMSE_sideall_newitem_k40 = np.sqrt(np.mean( (test_new_items.PredictedAll_40 - test_new_items.Rating)**2))
RMSE_sideall_newitem.append(RMSE_sideall_newitem_k40)
print("RMSE (side info, new item), k=40: ",  RMSE_sideall_newitem_k40)
#k=50, all side info
%%time
model_item_info_all = CMF(k=50, k_main=15, k_item=10, reg_param=1e-3, w_main=10.0, w_item=0.5, random_seed=1)
model_item_info_all.fit(deepcopy(train), item_info = deepcopy(item_side_info))
test_warm_start['PredictedAll_50'] = model_item_info_all.predict(test_warm_start.UserId, test_warm_start.ItemId)
test_new_items['PredictedAll_50'] = model_item_info_all.predict(test_new_items.UserId, test_new_items.ItemId)
RMSE_sideall_warm_k50 = np.sqrt(np.mean( (test_warm_start.PredictedAll_50 - test_warm_start.Rating)**2))
RMSE_sideall_warm.append(RMSE_sideall_warm_k50)
print("RMSE (side info, warm start), k=50: ",  RMSE_sideall_warm_k50)
RMSE_sideall_newitem_k50 = np.sqrt(np.mean( (test_new_items.PredictedAll_50 - test_new_items.Rating)**2))
RMSE_sideall_newitem.append(RMSE_sideall_newitem_k50)
print("RMSE (side info, new item), k=50: ",  RMSE_sideall_newitem_k50)<jupyter_output>RMSE (side info, new item), k=50:  1.3204711087663736<jupyter_text>5. Parameters Tunning<jupyter_code>K = [35, 40, 50]
print(RMSE_sideall_warm)
print(RMSE_sideall_newitem)
RMSE_noside = [RMSE_noside[2],RMSE_noside[0],RMSE_noside[1]]

print(RMSE_noside)
plt.plot(K, RMSE_sideall_warm, label ="with item side info")
plt.plot(K, RMSE_noside, label ="no item side info")
plt.xlabel('K')
plt.ylabel('RMSE')
plt.title('warm start RMSE comparison')
plt.legend()
plt.show()
plt.plot(K, RMSE_sideall_newitem, label = "new item")
plt.xlabel('K')
plt.ylabel('RMSE')
plt.title('RMSE with full side info - new-item')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>6. Prediction for users last reviews<jupyter_code>#sample train reviews data
link = "https://drive.google.com/open?id=16YyX6o1AUr1VNYB1DXnybxqqfA8Q-u1n"
_, drive_id = link.split('=')
downloaded = drive.CreateFile({'id': drive_id})
downloaded.GetContentFile('./sample_train.csv')
sample_train = pd.read_csv('./sample_train.csv')
print(len(sample_train))
sample_train.head()
#sample test reviews data
link = "https://drive.google.com/open?id=1P0p98XkIXW9QBVMDXFEiJMoaE8YsesWQ"
_, drive_id = link.split('=')
downloaded = drive.CreateFile({'id': drive_id})
downloaded.GetContentFile('./sample_test.csv')
sample_test = pd.read_csv('./sample_test.csv')
print(len(sample_test))
sample_test.head()
#Data processing
sample_train = sample_train.rename(columns = {'user_id': 'UserId', 'business_id': 'ItemId', 'rating':'Rating'})
sample_test = sample_test.rename(columns = {'user_id': 'UserId', 'business_id': 'ItemId', 'rating':'Rating'})
train_final = sample_train[['UserId', 'ItemId', 'Rating']].reset_index(drop=True)
test_final = sample_test[['UserId', 'ItemId', 'Rating']].reset_index(drop=True)
items_train_final = train_final.ItemId
items_test_final = test_final.ItemId
test_warm_start_final = test_final.loc[test_final.ItemId.isin(items_train_final)].reset_index(drop=True)
test_new_items_final = test_final.loc[~test_final.ItemId.isin(items_train_final)].reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>6.1 Make Prediction and Evaluate Result<jupyter_code>#k=50, all side info, for warm start
%%time
model_item_info_all = CMF(k=50, k_main=15, k_item=10, reg_param=1e-3, w_main=10.0, w_item=0.5, random_seed=1)
model_item_info_all.fit(deepcopy(train_final), item_info = deepcopy(item_side_info))
test_warm_start_final['PredictedAll'] = model_item_info_all.predict(test_warm_start_final.UserId, test_warm_start_final.ItemId)
RMSE_sideall_warm_final = np.sqrt(np.mean( (test_warm_start_final.PredictedAll - test_warm_start_final.Rating)**2))
MAE_sideall_warm_final = np.mean(np.abs(test_warm_start_final.PredictedAll - test_warm_start_final.Rating))

print("MAE (side info, warm start, final), k=50: ",  MAE_sideall_warm_final)
print("RMSE (side info, warm start, final), k=50: ",  RMSE_sideall_warm_final)
#k=40, all side info, for new Item
%%time
model_item_info_all = CMF(k=40, k_main=15, k_item=10, reg_param=1e-3, w_main=10.0, w_item=0.5, random_seed=1)
model_item_info_all.fit(deepcopy(train_final), item_info = deepcopy(item_side_info))
test_new_items_final['PredictedAll'] = model_item_info_all.predict(test_new_items_final.UserId, test_new_items_final.ItemId)
RMSE_sideall_newitem_final = np.sqrt(np.mean( (test_new_items_final.PredictedAll - test_new_items_final.Rating)**2))
MAE_sideall_newitem_final = np.mean(np.abs(test_new_items_final.PredictedAll - test_new_items_final.Rating))

print("MAE (side info, warm start, final), k=50: ",  RMSE_sideall_newitem_final)
print("RMSE (side info, warm start, final), k=50: ", MAE_sideall_newitem_final)
MAE_final = len(test_warm_start_final)/len(test_final)*MAE_sideall_warm_final + len(test_new_items_final)/len(test_final)*MAE_sideall_newitem_final
RMSE_final = len(test_warm_start_final)/len(test_final)*RMSE_sideall_warm_final + len(test_new_items_final)/len(test_final)*RMSE_sideall_newitem_final
print ("MAE is :", MAE_final)
print("RMSE is :", RMSE_final)<jupyter_output>MAE is : 1.233200947852274
RMSE is : 1.453980897864519<jupyter_text>6.2 Coverage Ratio<jupyter_code>#k=50
model_item_info_all_50 = CMF(k=50, k_main=15, k_item=10, reg_param=1e-3, w_main=10.0, w_item=0.5, random_seed=1)
model_item_info_all_50.fit(deepcopy(train_final), item_info = deepcopy(item_side_info))
train_final['PredictedAll'] = model_item_info_all_50.predict(train_final.UserId, train_final.ItemId)
predicted = pd.concat([test_new_items_final, test_warm_start_final, train_final])
print(len(predicted))
predicted.head()
def get_users_predictions(user_id, n, model):
    recommended_items = model[model['UserId'] == user_id][['PredictedAll', 'ItemId']]
    recommended_items = recommended_items.sort_values('PredictedAll', ascending=False)    
    if len(recommended_items) >= 10:
        recommended_items = recommended_items.head(n)        
    return recommended_items['ItemId'].tolist()

new = predicted['UserId'].unique()[:3000]
business_idx = list()
for i in new:
    business_idx.extend(get_users_predictions(i, 10, predicted))
Coverage_ratio_cmf = len(set(business_idx))/len(predicted[predicted['UserId'].isin (new)].ItemId.unique())
Coverage_ratio_cmf<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12454087879200108, 'fraction_numerical': 0.029995918922595566, 'mean_word_length': 8.382897255903, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 38, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '349235', 'license': 'MIT', 'n_tokens_mistral': 5988, 'n_tokens_neox': 5644, 'n_words': 877, 'path': 'notebooks/Collective Matrix Factorization Model.ipynb', 'repo_name': 'nding17/YelpRecommendation', 'size': '14702'}