<jupyter_start><jupyter_text>Latent Factor Analysis for Book Recommendationusing the [Book Crossing Datset](http://www2.informatik.uni-freiburg.de/~cziegler/BX/) and [**surprise**](http://surpriselib.com/) Preliminary<jupyter_code>import pandas as pd
import numpy as np
import time

from surprise import SVDpp
from surprise import Dataset, Reader, SVD, evaluate
from surprise import accuracy
from surprise.model_selection import train_test_split
from surprise.model_selection import GridSearchCV
from surprise.model_selection import cross_validate<jupyter_output><empty_output><jupyter_text>Data Wrangling<jupyter_code>raw_path = '../data/BX-Book-Ratings.csv'
raw = pd.read_csv(raw_path, 
                  sep=';',
                  header = 0,
                  names = ['user', 'isbn', 'rating'],
                  encoding = 'ISO-8859-1')
raw.head()
bookfile_path = '../data/BX-Books.csv'
df_book = pd.read_csv( bookfile_path,
                      sep = ';',
                      header = 0,
                      error_bad_lines = False,
                      usecols = [0,1,2],
                      index_col = 0,
                      names = ['isbn', 'title', 'author'],
                      encoding = 'iso-8859-1'
                     )
df_book.head()<jupyter_output><empty_output><jupyter_text>Build util function GetUserRatedBooks() and GetBookDetails()<jupyter_code>def GetUserRatedBooks( user_id, return_rating = False):
    udata = df_rating_clean[ df_rating_clean.user == user_id]
    if return_rating:
        return {udata.iloc[i].isbn : udata.iloc[i].rating for i in range(len(udata))}
    else:
        return list(udata['isbn'])

def GetBookDetails( isbn, field = 'title'):
    return str(df_book[df_book.index == isbn][field].values)
GetUserRatedBooks(276847, True)
GetBookDetails('0195153448')<jupyter_output><empty_output><jupyter_text>Remove books without informations<jupyter_code>missing_books = raw[-raw['isbn'].isin(df_book.index)]
df_rating = raw[raw['isbn'].isin(df_book.index)]
print( f'Found {len(missing_books)} missing books: {"{:.2%}".format(len(missing_books)/len(raw))}')<jupyter_output>Found 118605 missing books: 10.32%<jupyter_text>Remove Books and Users with limited ratingswe are setting a **threshold** here of minimum 10 ratings  users or books that has fewer than 10 ratings are simply not worth the effort<jupyter_code>books_ratings_count = df_rating.isbn.value_counts()
users_ratings_count = df_rating.user.value_counts()
rating_threshold = 10
books_tokeep = books_ratings_count[books_ratings_count>= rating_threshold]
users_tokeep = users_ratings_count[users_ratings_count>= rating_threshold]
print(f'Only {"{:.2%}".format(len(books_tokeep)/ len(books_ratings_count))} books will be kept')
print(f'Only {"{:.2%}".format(len(users_tokeep)/ len(users_ratings_count))} users will be kept')
      
print(f'There are {"{:,}".format(len(users_tokeep))} users and {"{:,}".format(len(books_tokeep))} books.')
df_rating_clean = df_rating[ df_rating.isbn.isin(books_tokeep.index)]
df_rating_clean = df_rating_clean[ df_rating_clean.user.isin(users_tokeep.index)]
print( f'We have {"{:,}".format(len(df_rating))} ratings before applying threshold, now we have {"{:,}".format(len(df_rating_clean))}')<jupyter_output>We have 1,031,175 ratings before applying threshold, now we have 428,085<jupyter_text>Remove Zeros<jupyter_code>zeros = df_rating_clean[df_rating_clean.rating == 0]
print( f'Found {len(zeros)} records that is 0: {"{:.2%}".format(len(zeros)/len(df_rating_clean))}')
df_rating_clean = df_rating_clean[df_rating_clean.rating != 0]
df_rating_clean.shape<jupyter_output><empty_output><jupyter_text>Sparseness of our Rating Matrix<jupyter_code>count_users = len(df_rating_clean.user.unique())
count_items = len(df_rating_clean.isbn.unique())
count_ratings = len(df_rating_clean)
print(f'We have {count_users} users X {count_items} items and {count_ratings} ratings. Resulting in a rating matrix that has a density of {"{:.2%}".format(count_ratings/ (count_users * count_items))}')<jupyter_output>We have 10807 users X 17091 items and 143663 ratings. Resulting in a rating matrix that has a density of 0.08%<jupyter_text>Models  Model Parameters<jupyter_code>n_factors = 10
n_epochs = 30
lr = 0.002
random_state = 420
verbose = False

test_userid = 276847<jupyter_output><empty_output><jupyter_text>Loading the dataset<jupyter_code>s_reader = Reader( rating_scale= (1,10))
# data have to follow this structure:
#   user: item: rating
s_data = Dataset.load_from_df(df_rating_clean[['user', 'isbn', 'rating']], s_reader)<jupyter_output><empty_output><jupyter_text>Using Surprise's SVD<jupyter_code># Train-Test Split
trainset, testset = train_test_split(s_data, test_size=0.1)

# define the model
svd = SVD(n_factors, n_epochs, lr_all = lr, random_state= random_state, verbose= verbose)

# Fit + Test-Predict
s_time = time.time()
svd.fit(trainset)
e_time = time.time()
print(f'Model building took {"{:.2f}".format((e_time - s_time)/60)} minutes')

test_predict = svd.test(testset)
print("Model accuracy using SVD : Test Set")
accuracy.rmse(test_predict, verbose=True)
def GetPredictions( trained_model, user_id, item_ids, org_df, verbose = False, top_n = None):
    if not org_df.user.isin([user_id]).any():
        print(f'{user_id} not in trained dataset. Cannot be predicted.')
        return None
    
    pred_df = org_df[ org_df.user == user_id]
    if verbose:
        s_time = time.time()
        print(f'user {user_id} had {len(pred_df)} ratings')

    compute_count = 1
    for item in item_ids:
        if not pred_df.isbn.isin( [item] ).any():
            ipred = trained_model.predict( uid= user_id, iid = item)
            ipred_data = {'user': user_id, 'isbn': item, 'rating': ipred.est}
            pred_df = pred_df.append( ipred_data, ignore_index = True)
            compute_count +=1
    
    if verbose:
        e_time = time.time()
        print(f'computed prediction for {compute_count} items.\n Took {e_time - s_time} seconds.')
        
    pred_df = pred_df.sort_values( by = ['rating'], ascending = False)
    
    if top_n:
        old_books = GetUserRatedBooks(user_id, return_rating= False)
        return pred_df[-pred_df.isbn.isin(old_books)][:top_n]
    else:
        return pred_df<jupyter_output><empty_output><jupyter_text>Let's Make a Prediction<jupyter_code>user_pred = GetPredictions( svd , test_userid, df_rating_clean.isbn.unique(), df_rating_clean, 
                          top_n = 10, verbose = True)
old_books = GetUserRatedBooks(test_userid, return_rating= True)
print(f'User {test_userid} already read:\n---------------------')
for ibook in old_books:
    bname = GetBookDetails(ibook)
    print(f'{str(bname)}: {old_books[ibook]}')

print(f'\nWe recommend:\n---------------------')
for ibook in user_pred.isbn:
    bname = GetBookDetails(ibook)
    print(f'{str(bname)}')<jupyter_output>User 276847 already read:
---------------------
['Nordermoor']: 8
['Der Kleine Hobbit']: 10
['Auf Ehre und Gewissen. Roman.']: 10
['Asche zu Asche.']: 7
['Denn sie betrÃ?Â¼gt man nicht.']: 8
['Mit dem KÃ?Â¼hlschrank durch Irland.']: 10
['Die HirnkÃ?Â¶nigin.']: 7
['<NAME> und der Stein der Weisen']: 10
['<NAME> und die Kammer des Schreckens']: 10
['<NAME> und der Gefangene von Azkaban']: 10
['<NAME> Und Der Feuerkelch']: 10

We recommend:
---------------------
['The Return of the King (The Lord of the Rings, Part 3)']
['To Kill a Mockingbird']
['<NAME> and the Order of the Phoenix (Book 5)']
['<NAME> and the Goblet of Fire (Book 4)']
["<NAME> and the Sorcerer's Stone (Harry Potter (Paperback))"]
['<NAME> and the Prisoner of Azkaban (Book 3)']
['The Fellowship of the Ring (The Lord of the Rings, Part 1)']
["<NAME> and the Sorcerer's Stone (Book 1)"]
['<NAME> and the Chamber of Secrets (Book 2)']
['<NAME> and the Prisoner o[...]<jupyter_text>Using Surprise SVD++<jupyter_code># define the model
svd_pp = SVDpp(n_factors, n_epochs, lr_all = lr, random_state= random_state, verbose= verbose)

# Fit + Test-Predict
s_time = time.time()
svd_pp.fit(trainset)
e_time = time.time()
print(f'Model building took {"{:.2f}".format((e_time - s_time)/60)} minutes')

test_predict_pp = svd_pp.test(testset)
print("Model accuracy using SVD++ : Test Set")
accuracy.rmse(test_predict_pp, verbose=True)<jupyter_output>Model building took 5.20 minutes
Model accuracy using SVD++ : Test Set
RMSE: 1.5683<jupyter_text>Make Prediction Again<jupyter_code>user_pred_pp = GetPredictions( svd_pp , test_userid, df_rating_clean.isbn.unique(), df_rating_clean, 
                          top_n = 10, verbose = True)
old_books = GetUserRatedBooks(test_userid, return_rating= True)
print(f'User {test_userid} already read:\n---------------------')
for ibook in old_books:
    bname = GetBookDetails(ibook)
    print(f'{str(bname)}: {old_books[ibook]}')

print(f'\nWe recommend:\n---------------------')
for ibook in user_pred_pp.isbn:
    bname = GetBookDetails(ibook)
    print(f'{str(bname)}')<jupyter_output>User 276847 already read:
---------------------
['Nordermoor']: 8
['Der Kleine Hobbit']: 10
['Auf Ehre und Gewissen. Roman.']: 10
['Asche zu Asche.']: 7
['Denn sie betrÃ?Â¼gt man nicht.']: 8
['Mit dem KÃ?Â¼hlschrank durch Irland.']: 10
['Die HirnkÃ?Â¶nigin.']: 7
['<NAME> und der Stein der Weisen']: 10
['<NAME> und die Kammer des Schreckens']: 10
['<NAME> und der Gefangene von Azkaban']: 10
['<NAME> Und Der Feuerkelch']: 10

We recommend:
---------------------
['<NAME> and the Goblet of Fire (Book 4)']
['Weirdos From Another Planet!']
['The Return of the King (The Lord of the Rings, Part 3)']
['Dune (Remembering Tomorrow)']
["The Time Traveler's Wife"]
['<NAME> and the Chamber of Secrets Postcard Book']
['To Kill a Mockingbird']
["My Sister's Keeper : A Novel (Picoult, Jodi)"]
['The Little Prince']
["Charlotte's Web (Trophy Newbery)"]
{'fraction_non_alphanumeric': 0.14917014560635372, 'fraction_numerical': 0.01914265349760717, 'mean_word_length': 4.263665594855305, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '361063', 'license': 'MIT', 'n_tokens_mistral': 3652, 'n_tokens_neox': 3411, 'n_words': 850, 'path': 'latent_factor_analysis/LFA_surprise.ipynb', 'repo_name': 'youonf/recommendation_system', 'size': '9923'}