<jupyter_start><jupyter_text>Performing the normal sparse SVD and folding in as discussed in the research paper<jupyter_code># The required matrix but in df form. index is userid and columns are the movieids
# to use it as a matrix use utility_df.values

utility_df = ratings_df.pivot(
    index='userId',
    columns='movieId',
    values='rating'
).fillna(0)#.T.fillna(utility_df.mean(axis=1)).T

# R = utility_df.values
# user_ratings_mean = np.mean(R, axis = 1)
# R_demeaned = R - user_ratings_mean.reshape(-1, 1)
# utility_df = pd.DataFrame(R_demeaned, columns = utility_df.columns)
# utility_df = utility_df.fillna(0)
utility_df.values[1][39]
utility_mat[1][17]
utility_df = utility_df[:-1]
utility_df.shape
#userid - 1 will be the first index and movieid -1 will be the second index
new_user = (utility_df.values)[670]
R = sparse.csc_matrix(utility_df.values,dtype=float)
#perform SVD
U, Sigma, VT = randomized_svd(R,n_components=300,n_iter=5,random_state=None)
#U, Sigma, VT = svds(R, k=50)
#u,s,vt = svds(R_demeaned, k = 50)

#all_user_predicted_ratings = u.dot(np.diag(s).dot(vt))
all_user_predicted_ratings = U.dot(np.diag(Sigma).dot(VT))
all_user_predicted_ratings[320][0]
#convert the matrix to df same as our utility_df but this one is preds_df
preds_df = pd.DataFrame(all_user_predicted_ratings, columns = utility_df.columns)
preds_df.shape
preds_df.head()<jupyter_output><empty_output><jupyter_text>Completed the Collaborative Filtering part now lets Recommend<jupyter_code>def get_viewed_movies(userId) :
    ''' 
    aggregate all movies that the user watched 
    input : userid
    output : a list of movie ids that the user has already rated
    '''
    
    users_viewed_movies = ratings_df[ratings_df['userId'] == userId].sort_values(['rating'], ascending=False)
    
    
    #print(str(userId) +  " has viewed following films")
    
    #r = movies_df.merge(users_viewed_movies, left_on='id', right_on='movieId').sort_values(['rating'],ascending=False)
    
#     for row in users_viewed_movies.itertuples():
#         rating = row[1]
#         print(str(row[3]) + " " + str(row[2]) + '\t' + str(rating))
#    print(r.head())
    return users_viewed_movies['movieId'].tolist()
p = get_viewed_movies(321)
print(232 in p)
#ratings_df[ratings_df['userId'] == 321]
def recommend_movies(preds_df, userID, movies_df, original_ratings_df, num_recommendations=5):
    
    # Get and sort the user's predictions
    
    # UserID starts at 1, not 0
    user_row_number = userID - 1 
    
    sorted_user_predictions = preds_df.iloc[user_row_number].sort_values(ascending=False)
    print(sorted_user_predictions.head())
    
    # Get the user's data and merge in the movie information.
    viewed_movieids = get_viewed_movies(userID)
               
    all_movieids = sorted_user_predictions.index.values.tolist()
    predicted_movieids = [value for value in all_movieids if value not in viewed_movieids]
    #print(type(predicted_movieids))
    recommendations = movies_df[movies_df['id'].isin(predicted_movieids)]
    recommendations = recommendations.merge(sorted_user_predictions, left_on='id', right_index=True).sort_values([userID-1],ascending=False)
    return predicted_movieids,recommendations
pm , predictions = recommend_movies(preds_df, 321, movies_df,ratings_df, 10)
print(319 in pm)
predictions.head()
new_user = (utility_df.values)[670]
new_user.shape
def add_new_user_to_collab(new_user,U,Sigma,VT,all_user_predicted_ratings) :
    
    #sparse matrix -> U.Sigma.VT
    #q = sparse_user. V. Sigma^-1
    q = new_user.dot(VT.transpose().dot(np.linalg.inv(np.diag(Sigma))))
    
    U = np.append(U,[q],axis=0)
    
    new = q.dot(np.diag(Sigma).dot(VT))
    all_user_predicted_ratings = np.append(all_user_predicted_ratings,[new],axis=0) 
    preds_df = pd.DataFrame(all_user_predicted_ratings, columns = utility_df.columns)
    print(preds_df.shape)
    return U, all_user_predicted_ratings, preds_df
U, all_user_predicted_ratings, preds_df = add_new_user_to_collab(new_user,U,Sigma,VT,all_user_predicted_ratings)
p = get_viewed_movies(671)
print(1 in pm)
print(1 in p)
preds_df[671][]

pm , predictions = recommend_movies(preds_df, 671, movies_df,ratings_df, 10)
print(1136 in pm)
predictions.head()
#TODO : 
#add new ratings to rating_df
#implement the new item version
#create python scripts<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11539343886212435, 'fraction_numerical': 0.017894012388162423, 'mean_word_length': 5.063977746870654, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '189833', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1663, 'n_tokens_neox': 1599, 'n_words': 319, 'path': 'collaborative_filtering.ipynb', 'repo_name': 'ChaityaChheda/BTech-Project', 'size': '4359'}