<jupyter_start><jupyter_text>Django based movie recommender systemAuthor: <NAME>- Item_based CF filering & SVD decomposition RecSys<jupyter_code>import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error
from surprise import Dataset, Reader, SVD, accuracy
from surprise.model_selection import train_test_split 
from prettytable import PrettyTable
"""
using ml rating dataset (which has ideal number of ratings for this project)
"""
df_rating = pd.read_csv('https://raw.githubusercontent.com/xiaolancara/Recommender-System/main/data/ml-latest-small/ml-latest-small/ratings.csv').iloc[:,:3]
df_rating.columns = ['userid', 'movieid', 'ratings']
# get top 30 popular movies 
popular_movies = list(df_rating.movieid.value_counts()[:30].index)
df_rating = df_rating[df_rating['movieid'].isin(popular_movies)]
vc = df_rating['userid'].value_counts()
# get user that rated at least 5 movies
user_index = list(vc[vc>=5].index)
# subset ratings to 1000
df_rating = df_rating[df_rating.userid.isin(user_index)][:1000].reset_index(drop=True)
df_rating
# """
# using system's rating dataset in this project
# """
# df_rating = pd.read_csv('movie_rating.csv',index_col=0)
# df_rating.columns = ['id','userid', 'movieid', 'ratings']
# df_rating
df_rating.movieid.unique()
df_rating.userid.value_counts()<jupyter_output><empty_output><jupyter_text>Using Item_based collabrative filtering<jupyter_code>userRatings = df_rating.pivot_table(index=['userid'],columns=['movieid'],values='ratings')
userRatings
corrMatrix = userRatings.corr(method='pearson').fillna(0)
corrMatrix<jupyter_output><empty_output><jupyter_text>Predict ratings for itemsThe method I use to predict the rating for specific user and item is as below, where K is the neighborhood of most similar items rated by active user a, and w(i,j) is the similarity between items i and j.![image.png](attachment:b88900b0-84de-4d7e-b002-72c712d7770f.png)<jupyter_code>def evaluate():
    df_predict = pd.DataFrame()
    for user_i in userRatings.index:
        myRatings = userRatings.loc[user_i].dropna()

        for movie_i in list(myRatings.index):
            # retrieve similar movies for movie i
            similar_movies = corrMatrix[movie_i]
            # substract to similar score between movie i and rated movies
            similar_movies = similar_movies[similar_movies.index.isin(myRatings.index)]
            # calculate predict rating
            # adding 0.01 to avoid 0 similar score in low number of ratings system
            predict_ratings = sum(myRatings * similar_movies)/(sum(np.abs(similar_movies))+0.01)
            df_predict = df_predict.append([[user_i, movie_i,predict_ratings]])

    df_predict.columns = ['user_id','movie_id','predict_rating']
    df_predict.reset_index(drop=True,inplace=True)
    
    return df_predict

# evaluate rms
cf_rms = mean_squared_error(df_rating['ratings'], evaluate()['predict_rating'], squared=False)
cf_rms
 evaluate().head(10)
def recommend_movie(user_id, hm):
    myRatings = userRatings.loc[user_id].dropna()
    similar_candidates = pd.DataFrame()
    for i in list(corrMatrix.index):
        # retrieve similar movies for movie i
        similar_movies = corrMatrix[i]
        # substract to similar score between movie i and rated movies
        similar_movies = similar_movies[similar_movies.index.isin(myRatings.index)]
        # calculate predict rating
        predict_ratings = sum(myRatings * similar_movies)/(sum(np.abs(similar_movies))+0.01)
        similar_candidates = similar_candidates.append([predict_ratings])
    similar_candidates.index = corrMatrix.index
    # substract recommend movies that  have rated
    similar_candidates = similar_candidates[~similar_candidates.index.isin(myRatings.index)]
    similar_candidates.columns = ['predict_rating']
    similar_candidates.sort_values(by='predict_rating',inplace=True,ascending=False)    
    return similar_candidates[:hm]
# recommend movies for user_id 1
rec_movie = recommend_movie(user_id=1,hm=10)
rec_movie
# movies that user 1 has rated
myRatings = userRatings.loc[1].dropna()
myRatings
print('recommend movie id are', list(rec_movie.index))<jupyter_output>recommend movie id are [4993, 7153, 2762, 5952, 318, 858, 588, 589, 150]<jupyter_text>Using SVD model<jupyter_code># instantiate a reader and read in our rating data
reader = Reader(rating_scale=(1, 5))
data = Dataset.load_from_df(df_rating[['userid','movieid','ratings']], reader)

# train SVD on 75% of known rates
trainset, testset = train_test_split(data, test_size=.25, random_state=30)
algorithm = SVD()
algorithm.fit(trainset)
predictions = algorithm.test(testset)

# check the accuracy using Root Mean Square Error
svd_rms = accuracy.rmse(predictions)
print(svd_rms)
def pred_user_rating(ui):
    if ui in df_rating.userid.unique():
        ui_list = df_rating[df_rating.userid == ui].movieid.tolist()
        d = df_rating.movieid.unique()
        d = [v for v in d if not v in ui_list] 
        predictedL = []
        for j in d:     
            predicted = algorithm.predict(ui, j)
            predictedL.append((j, predicted[3])) 
        pdf = pd.DataFrame(predictedL, columns = ['movieid', 'ratings'])
        pdf.sort_values('ratings', ascending=False, inplace=True)  
        pdf.set_index('movieid', inplace=True)    
        return pdf.head(10)        
    else:
        print("User Id does not exist in the list!")
        return None
df_rating.userid.unique()
# recommend movies for user_id 1
rec_movie = pred_user_rating(1)
rec_movie
# movies that user 1 has rated
df_rating[df_rating.userid == 1]<jupyter_output><empty_output><jupyter_text>Merge them together<jupyter_code>class item_basedCF:
    def __init__(self):
        self.userRatings = df_rating.pivot_table(index=['userid'],columns=['movieid'],values='ratings')
        self.corrMatrix =  self.userRatings.corr(method='pearson').fillna(0)
        self.cf_rms = mean_squared_error(df_rating['ratings'],  self.evaluate()['predict_rating'], squared=False)
    def evaluate(self):
        df_predict = pd.DataFrame()
        for user_i in  self.userRatings.index:
            myRatings =  self.userRatings.loc[user_i].dropna()

            for movie_i in list(myRatings.index):
                # retrieve similar movies for movie i
                similar_movies =  self.corrMatrix[movie_i]
                # substract to similar score between movie i and rated movies
                similar_movies = similar_movies[similar_movies.index.isin(myRatings.index)]
                # calculate predict rating
                # adding 0.01 to avoid 0 similar score in low number of ratings system
                predict_ratings = sum(myRatings * similar_movies)/(sum(np.abs(similar_movies))+0.01)
                df_predict = df_predict.append([[user_i, movie_i,predict_ratings]])

        df_predict.columns = ['user_id','movie_id','predict_rating']
        df_predict.reset_index(drop=True,inplace=True)

        return df_predict

    def recommend_movie(self, user_id):
        myRatings =  self.userRatings.loc[user_id].dropna()
        similar_candidates = pd.DataFrame()
        for i in list( self.corrMatrix.index):
            # retrieve similar movies for movie i
            similar_movies =  self.corrMatrix[i]
            # substract to similar score between movie i and rated movies
            similar_movies = similar_movies[similar_movies.index.isin(myRatings.index)]
            # calculate predict rating
            predict_ratings = sum(myRatings * similar_movies)/(sum(np.abs(similar_movies))+0.01)
            similar_candidates = similar_candidates.append([predict_ratings])
        similar_candidates.index =  self.corrMatrix.index
        # substract recommend movies that  have rated
        similar_candidates = similar_candidates[~similar_candidates.index.isin(myRatings.index)]
        similar_candidates.columns = ['predict_rating']
        similar_candidates.sort_values(by='predict_rating',inplace=True,ascending=False)    
        return similar_candidates[:10]
class svd_based:
    def __init__(self):
        # instantiate a reader and read in our rating data
        reader = Reader(rating_scale=(1, 5))
        data = Dataset.load_from_df(df_rating[['userid','movieid','ratings']], reader)

        # train SVD on 75% of known rates
        trainset, testset = train_test_split(data, test_size=.25, random_state=30)
        self.algorithm = SVD()
        self.algorithm.fit(trainset)
        predictions =  self.algorithm.test(testset)

        # check the accuracy using Root Mean Square Error
        self.svd_rms = accuracy.rmse(predictions)
    def pred_user_rating(self, ui):
        if ui in df_rating.userid.unique():
            ui_list = df_rating[df_rating.userid == ui].movieid.tolist()
            d = df_rating.movieid.unique()
            d = [v for v in d if not v in ui_list] 
            predictedL = []
            for j in d:     
                predicted =  self.algorithm.predict(ui, j)
                predictedL.append((j, predicted[3])) 
            pdf = pd.DataFrame(predictedL, columns = ['movieid', 'ratings'])
            pdf.sort_values('ratings', ascending=False, inplace=True)  
            pdf.set_index('movieid', inplace=True)    
            return pdf.head(10)        
        else:
            print("User Id does not exist in the list!")
            return None
itembased_object = item_basedCF()
svdbased_object = svd_based()
# choose either one system by lower rmse 
if itembased_object.cf_rms > svdbased_object.svd_rms:
    movie_rec = svdbased_object.pred_user_rating(1)  # predict for user one
    print('svd based: ',svdbased_object.svd_rms)
else:
    movie_rec = itembased_object.recommend_movie(1)
    print('item based: ',itembased_object.cf_rms)
movie_rec<jupyter_output>RMSE: 0.8244
svd based:  0.8243650498175716<jupyter_text>MRR vs MAP vs NDCG Rank-Aware Evaluation Metrics Reference: https://medium.com/swlh/rank-aware-recsys-evaluation-metrics-5191bba16832FeedBack evaluation using test data.From the above recommended movies for user 5 and 4, we assumed they like some of them, and create the feeback test dataset as follows.<jupyter_code># create list to store the ranking evaluation result
list_item_ranking = []
list_svd_ranking = []
ranking_method = ['MRR', 'MAP','NDCG']<jupyter_output><empty_output><jupyter_text>In item based<jupyter_code>itembased_object.recommend_movie(5).index
itembased_object.recommend_movie(4).index<jupyter_output><empty_output><jupyter_text>We assume user5 likes movies 2028, 593, 7153; user4 likes movies 50 and 858.<jupyter_code>NaN = np.nan
user5_like = [2028, 593, 7153]
user4_like = [50, 858]
df_item_user5 = pd.DataFrame(NaN, index=itembased_object.recommend_movie(5).index, columns=['like'])
df_item_user5['like'] = df_item_user5[df_item_user5['like'].index.isin(user5_like)].fillna(1)
df_item_user5.reset_index(inplace = True)
df_item_user5.index += 1
# adding rating score for NDCG method calculation
rating_data = [4, 0, 4, 0, 3, 0, 2, 0, 0, 1 ]
df_item_user5['ratings'] = rating_data
df_item_user5
df_item_user4 = pd.DataFrame(NaN, index=itembased_object.recommend_movie(4).index, columns=['like'])
df_item_user4['like'] = df_item_user4[df_item_user4['like'].index.isin(user4_like)].fillna(1)
df_item_user4.reset_index(inplace = True)
df_item_user4.index += 1
# adding rating score for NDCG method calculation
rating_data = [0, 4, 0, 2, 0, 2, 0, 1, 0, 5 ]
df_item_user4['ratings'] = rating_data
df_item_user4<jupyter_output><empty_output><jupyter_text>MRR: Mean Reciprocal Rank![image.png](attachment:fddee87a-20e1-4854-ace2-d23df2308425.png)<jupyter_code>list_item_ranking.append(np.round((1/1 + 1/2)/2, 2))
list_item_ranking[-1]<jupyter_output><empty_output><jupyter_text>MAP: Average Precision and Mean Average Precision![image.png](attachment:93d54812-1aad-47f3-8fd3-1001fa93267c.png)<jupyter_code>list_item_ranking.append(np.round((((1/1+2/3+3/5)/3) + ((1/2+2/10)/2))/2,2))
list_item_ranking[-1]<jupyter_output><empty_output><jupyter_text>NDCG: Normalized Discounted Cumulative Gain![image.png](attachment:edd7bbbd-3f6a-4810-9aae-72edb4b4422e.png)<jupyter_code>DCG = 4/np.log2(2)+4/np.log2(4)+3/np.log2(6)+2/np.log2(8)+1/np.log2(11)
IDCG = 4/np.log2(2)+4/np.log2(3)+3/np.log2(4)+2/np.log2(5)+1/np.log2(6)
NDCG_user5 = DCG/IDCG

DCG = 4/np.log2(3)+2/np.log2(5)+2/np.log2(7)+1/np.log2(9)+5/np.log2(11)
IDCG = 5/np.log2(2)+4/np.log2(3)+2/np.log2(4)+2/np.log2(5)+1/np.log2(6)
NDCG_user4 = DCG/IDCG


list_item_ranking.append(np.round((NDCG_user5+NDCG_user4)/2,2))
list_item_ranking[-1]<jupyter_output><empty_output><jupyter_text>In svd based<jupyter_code>df_svd_user5 = pd.DataFrame(NaN, index=svdbased_object.pred_user_rating(5).index, columns=['like'])
df_svd_user5['like'] = df_svd_user5[df_svd_user5['like'].index.isin(user5_like)].fillna(1)
df_svd_user5.reset_index(inplace = True)
df_svd_user5.index += 1
# adding rating score for NDCG method calculation
rating_data = [4, 0, 0, 4, 1, 3, 0, 0, 0, 2 ]
df_svd_user5['ratings'] = rating_data
df_svd_user5
df_svd_user4 = pd.DataFrame(NaN, index=svdbased_object.pred_user_rating(4).index, columns=['like'])
df_svd_user4['like'] = df_svd_user4[df_svd_user4['like'].index.isin(user4_like)].fillna(1)
df_svd_user4.reset_index(inplace = True)
df_svd_user4.index += 1
# adding rating score for NDCG method calculation
rating_data = [2, 4, 0, 2, 5, 0, 0, 1, 0, 0 ]
df_svd_user4['ratings'] = rating_data
df_svd_user4
# MRR
list_svd_ranking.append(np.round((1/4+1/7)/2,2))
list_svd_ranking[-1]
# MAP
list_svd_ranking.append(np.round(((1/4+2/7+3/8)/3 + (1/7+2/10)/2)/2, 2))
list_svd_ranking[-1]
# NDCG
DCG = 4/np.log2(2)+4/np.log2(5)+1/np.log2(6)+3/np.log2(7)+2/np.log2(11)
IDCG = 4/np.log2(2)+4/np.log2(3)+3/np.log2(4)+2/np.log2(5)+1/np.log2(6)
NDCG_user5 = DCG/IDCG

DCG = 2/np.log2(2)+4/np.log2(3)+2/np.log2(5)+5/np.log2(6)+1/np.log2(9)
IDCG = 5/np.log2(2)+4/np.log2(3)+2/np.log2(4)+2/np.log2(5)+1/np.log2(6)
NDCG_user4 = DCG/IDCG


list_svd_ranking.append(np.round((NDCG_user5+NDCG_user4)/2,2))
list_svd_ranking[-1]<jupyter_output><empty_output><jupyter_text>Ranking Evaluation Result<jupyter_code>t = PrettyTable(['ranking_method','item', 'svd'])
for i in range(len(list_item_ranking)):
    t.add_row([ranking_method[i], list_item_ranking[i], list_svd_ranking[i]])
print(t)<jupyter_output>+----------------+------+------+
| ranking_method | item | svd  |
+----------------+------+------+
|      MRR       | 0.75 | 0.2  |
|      MAP       | 0.55 | 0.24 |
|      NDCG      | 0.74 | 0.81 |
+----------------+------+------+
{'fraction_non_alphanumeric': 0.13004330194515087, 'fraction_numerical': 0.037390885971544434, 'mean_word_length': 4.220667384284177, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 53, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '518016', 'license': 'MIT', 'n_tokens_mistral': 5566, 'n_tokens_neox': 5137, 'n_words': 966, 'path': 'Algorithm- Item_based & SVD.ipynb', 'repo_name': 'xiaolancara/cara_movie_recommender', 'size': '14560'}