<jupyter_start><jupyter_text>Recommender System based on Regularized Matrix Factorization<jupyter_code># Data manipulation
import pandas as pd
import numpy as np
import numba
from lenskit import batch, topn, util
from lenskit import crossfold as xf
from lenskit.algorithms import Recommender, Predictor, als, basic, user_knn
from lenskit import topn
from scipy import sparse
from sklearn.metrics.pairwise import cosine_similarity
from lenskit.data import sparse_ratings

# Dataset
from lenskit.datasets import ML100K
movielens = ML100K('../ml-100k')

# Options for pandas
pd.options.display.max_columns = 50
pd.options.display.max_rows = 30

# Visualizations and debugging
import plotly.graph_objs as go
#from pprintpp import pprint as pp
import logging

# Autoreload extension
if 'autoreload' not in get_ipython().extension_manager.loaded:
    %load_ext autoreload
    
%autoreload 2
from lenskit.datasets import ML100K
movielens = ML100K('ml-100k')
# Data import
ratings = movielens.ratings
uir, users, items = sparse_ratings(ratings, scipy=True)
M = uir
lambda1,lambda2,tol = 1.0,10.0,1e-3
X_o=csr_matrix(M.shape)
Z=csr_matrix(M.shape)
Gamma=csr_matrix(M.shape)
M_s = (uir!=0)
from scipy.sparse.linalg import svds, eigs, norm
from scipy.sparse import csr_matrix
from numpy import linalg as LA
from numba import jit, njit, prange
#@njit(parallel=True)
@jit(nopython=True)
def matrix_factorization_pred(X,P,Q,K,steps,alpha,beta,Mask):
#    Mask = (X!=0)
    Q = Q.T
    error_list = np.zeros(steps)
    for step in range(steps):
        print(step)
        #for each user
        for i in prange(X.shape[0]):
            #for each item
            for j in range(X.shape[1]):
                if X[i,j] > 0 :

                    #calculate the error of the element
                    eij = X[i,j] - np.dot(P[i,:],Q[:,j])
                    #second norm of P and Q for regularilization
                    sum_of_norms = 0
                    #for k in xrange(K):
                    #    sum_of_norms += LA.norm(P[:,k]) + LA.norm(Q[k,:])
                    #added regularized term to the error
                    sum_of_norms += LA.norm(P) + LA.norm(Q)
                    #print sum_of_norms
                    eij += ((beta/2) * sum_of_norms)
                    #compute the gradient from the error
                    for k in range(K):
                        P[i][k] = P[i][k] + alpha * ( 2 * eij * Q[k][j] - (beta * P[i][k]))
                        Q[k][j] = Q[k][j] + alpha * (2 * eij * P[i][k] - (beta * Q[k][j]))

        #compute total error
        error = 0
        #for each user
        extimated_X = np.trunc(P @ Q)
        extimated_X = np.where(extimated_X>5, 5, extimated_X)
        extimated_X = np.where(extimated_X<0, 0, extimated_X)
        extimated_error = np.multiply(X - extimated_X, Mask)
        error = LA.norm(extimated_error)
        error_list[step] = error
        
        if error < 0.001:
            break
    return extimated_X, P, Q.T, error_list
class MatrixFactorization(Recommender, Predictor):
    """
    Recommend new items by completing the user-item rating matrix with regularized nonnegative matrix factorization
    """
    def __init__(self, K=8, steps=300, alpha=0.0002, beta=float(0.02), selector = None):
        
        # Set selector
        if selector is None:
            self.selector = basic.UnratedItemCandidateSelector()
        else:
            self.selector = selector
            
        # Set parameters
        self.steps = steps
        self.K = K
        self.alpha = alpha
        self.beta = beta
        
        # Enable logging 
        _logger = logging.getLogger(__name__)

    def __str__(self):
        return 'MatrixFactorization'
            
    # Store the ratings matrix in sparse format and generate similarity matrix
    def fit(self, ratings, **kwargs):
        
        # Get sparse representation in CSR format
        uir, users, items = sparse_ratings(ratings, scipy=True)
        
        # Store ratings
        self.rating_matrix_ = uir
        self.user_index_ = users
        self.item_index_ = items
        # Calculate mask matrix from rating matrix
        self.mask_matrix = (self.rating_matrix_!=0)
            
        # Calculate similarites from sparse matrix
######        self.sim_matrix_ = cosine_similarity(self.rating_matrix_)
        
        # Calculate completed u-i matrix matrix with matrix factorization
                #Grab the input rating matrix and mask matrix
        M = self.rating_matrix_
        M_s = self.mask_matrix
        #P: an initial matrix of dimension N x K, where is n is no of users and k is hidden latent features
        P = np.random.rand(M.shape[0],self.K)
        #Q : an initial matrix of dimension M x K, where M is no of movies and K is hidden latent features
        Q = np.random.rand(M.shape[1],self.K)
        
        self.full_matrix_,_,_,_ = matrix_factorization_pred(M.todense(), P, Q, self.K, self.steps, self.alpha, self.beta, M_s.todense())               
        
        # Reduce candidate space to unseen items
        self.selector.fit(ratings)

            # Add a user to the ratings matrix
    def add_user(self, user_id):
        
        # Check if user_id to be added already exists
        try:
            assert (user_id in self.user_index_) == False, "User ID already exists! Not adding anything..."
        
        except AssertionError as e:
            print(e)
            exit(1)

        # Build a sparse matrix of length of number of items
        tmp_sparse_row = sparse.csr_matrix(np.zeros((1,len(self.item_index_))))

        # Vertically stack temporary matrix to original matrix
        self.rating_matrix_ = sparse.vstack([self.rating_matrix_, tmp_sparse_row])
        
        # Update user index
        self.user_index_ = self.user_index_.append(pd.Index([user_id]))
    
        
    # Add a user to the ratings matrix
    def add_item(self, item_id):
        
        # Check if item_id to be added already exists
        try:
            assert (item_id in self.item_index_) == False, "Item ID already exists!"
        
        except AssertionError as e:
            print(e)
            exit(1)
        
        # Build a sparse matrix of length of number of users
        tmp_sparse_col = sparse.csr_matrix(np.zeros((len(self.user_index_),1)))
        
        # Horizotnally stack temporary matrix to original matrix
        self.rating_matrix_ = sparse.hstack([self.rating_matrix_, tmp_sparse_col])
        
        # Update item index
        self.item_index_ = self.item_index_.append(pd.Index([item_id]))
        
        
    # Add a user-item interaction for existing users and items
    def add_interactions(self, user_id, item_id, rating):
    
        # Check if inputs are lists and all input list lengths are equal
        assert type(user_id) == list, "Input user_id is not a list"
        assert type(item_id) == list , "Input item_id is not a list"
        assert type(rating) == list, "Input rating is not a list"
        assert len(user_id) == len(item_id) == len(rating), "Input lists are not of the same length"
        
        # Build a temporary sparse LIL matrix
        
        tmp_ratings = sparse.lil_matrix(self.rating_matrix_.shape)
        
        for i in range(len(user_id)):
            
            # Obtain locations from ID
            user_pos, = np.where(self.user_index_ == user_id[i])[0]
            item_pos, = np.where(self.item_index_ == item_id[i])[0]
            
            # Fill into temporary sparse matrix
            tmp_ratings[user_pos, item_pos] = rating[i]
                    
        # Convert temporary LIL to CSR
        tmp_ratings = tmp_ratings.tocsr()
        
        # Add temporary CSR to main ratings matrix
        self.rating_matrix_ += tmp_ratings

    # Provide a recommendation of top "n" movies given "user"
    # The recommender uses the UnratedItemCandidateSelector by default and uses the ratings matrix 
    # it was originally fit on
    def recommend(self, user_id, candidates=None, ratings=None):
        
        # Reduce candidate space and store candidates with item ID
        if candidates is None:
            candidates = self.selector.candidates(user_id, ratings)
        
        # Grab user index for given user_id
        user_index, = np.where(self.user_index_ == user_id)[0]
                
        # Predict ratings and scores for all unseen items
        prediction_score_df = self.predict_for_user(user_index, self.full_matrix_, candidates)
        
        return(prediction_score_df)
    
    
    def predict_for_user(self, user, extimated_X, items):
        
        # Instantiate ratings and item_popularity vectors
        predicted_ratings = np.zeros(len(items), dtype=float)
        item_popularity = np.zeros(len(items), dtype=float)
        
        # Convert ratings matrix to COO matrix
        coo_ratings = self.rating_matrix_.tocoo()
        rating_matrix_users = coo_ratings.row
        rating_matrix_items = coo_ratings.col
        rating_matrix_data = coo_ratings.data                
            
        # For each unseen item
        for i in range(len(items)):
                        
            # Item position given item i ID
            item_pos = self.item_index_.get_loc(items[i])
            
            # Locations of ratings for item_pos
            rating_locations, = np.where(rating_matrix_items == item_pos)
            
            # Store popularity of item based on number of total ratings 
            item_popularity[i] = len(rating_locations)
            
            #predicted_ratings[i] = extimated_X[user_pos,item_pos]
            predicted_ratings[i] = extimated_X[user,item_pos]
            
        # minmax scale the popularity of each item
        normalized_popularity = np.interp(item_popularity, (item_popularity.min(), item_popularity.max()), (0, +1))
        score = np.multiply(normalized_popularity, predicted_ratings)
        
        results = {'predicted_ratings':predicted_ratings, 'normalized_popularity':normalized_popularity}
        return pd.DataFrame(results, index=items)
            
        return None
%%time

# Instantiate object
algo_mf = MatrixFactorization()

# Reduce the candidates space + build user-user cosin similarity matrix 
algo_mf.fit(ratings)
%%time

# Ask for rating predictions for u users
for u in range (1,10):
    recs = algo_mf.recommend(u)
# View the last set of recommendations
recs.sort_values(
    by=["predicted_ratings", "normalized_popularity"],
    ascending=False
)[["predicted_ratings", "normalized_popularity"]].head(20)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08483483483483484, 'fraction_numerical': 0.007413663663663664, 'mean_word_length': 2.257029339853301, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '241389', 'license': 'MIT', 'n_tokens_mistral': 3278, 'n_tokens_neox': 3153, 'n_words': 902, 'path': 'notebooks/MF.ipynb', 'repo_name': 'haloship/rec-sys-dynamics', 'size': '10656'}