<jupyter_start><jupyter_text>load the split dataset into train and test.build the Iu and Ui data structure<jupyter_code>with open("ratebeer_train_random.json","r") as infile:
    train = json.load(infile)
infile.close()
with open("ratebeer_test_random.json","r") as infile:
    test = json.load(infile)
infile.close()
with open("ratebeer_quickmap_random.json","r") as infile:
    quickmap = json.load(infile)
infile.close()

print(len(train),len(test),len(quickmap))
Iu = dict() #set of products reviewed by users
Ui = dict() #set of users who reviewed the product
train = sorted(train, key = lambda k : k["review/time"])
for review in train:
        item = review["product/productId"]
        user = review["review/userId"]
        if item in Ui:
            Ui[item].append(user)
        else:
            Ui[item] = [user]
        if user in Iu:
            Iu[user].append(item)
        else:
            Iu[user] = [item]<jupyter_output><empty_output><jupyter_text>Construct user and item mapping to integer indices<jupyter_code>distinct_user_set = set()
distinct_item_set = set()
for review in train:
    if review["review/userId"] not in distinct_user_set:
        distinct_user_set.add(review["review/userId"])
    if review["product/productId"] not in distinct_item_set:
        distinct_item_set.add(review["product/productId"])
print(len(distinct_user_set), len(distinct_item_set))
import sys
sys.setrecursionlimit(20000)<jupyter_output><empty_output><jupyter_text>Latent factor model  -standard<jupyter_code>class LFM(object):
    ''' LFM class implements the standard latent factor model of collaborative filtering using matrix factorization
    '''
    
    def __init__(self,train_data, Iu_reg, Ui_reg, userproduct_dict,userset,itemset,k,Lambda):
        ''' requires Iu and Ui matrix information, quick mapping of reviews to (user,product) and k for set up
        also sets up k fold training data for k-cross validation
        '''
        
        self.Ntrain = len(train_data)               #Number of training samples
        self.train_data = train_data                #the train_data used
        self.Iu = self.deepish_copy(Iu_reg)         #Iu mapping
        self.Ui = self.deepish_copy(Ui_reg)         #Ui mapping
        self.quickmap = userproduct_dict            #uses key as (userid-itemid) for quick mapping to required review
        self.user_set = userset
        self.item_set = itemset
        
        self.user_map = {}
        self.item_map = {}
        i=0
        for user in self.user_set:
            self.user_map[i] = user
            i+=1
        i=0
        for item in self.item_set:
            self.item_map[i] = item
            i+=1
            
        #hyperparameters
        self.Lambda= Lambda               #regularization param
        self.k=k                          # number of latent factor dimension (low dimensional repr)
        self.final_param = self.init_theta()  #initialize the current final_param to some value.
        
        
    def init_theta(self):
        ''' Initializes the parameter of the standard model
        flat_theta = <alpha, Bu, Bi, Gu,Gu>
        '''
        flat_theta = []
        #baseline predictors
        self.alpha = 0                    #global offset
        flat_theta.append(self.alpha)
        
        self.Bu = dict()                  #user bias initialized randomly
        self.Bi = dict()                  #item bias initialized randomly
        for i in range(len(self.user_map)):
            self.Bu[self.user_map[i]] =np.random.random(1).item() 
            flat_theta.append(self.Bu[self.user_map[i]])
        for i in range(len(self.item_map)):
            self.Bi[self.item_map[i]] = np.random.random(1).item() 
            flat_theta.append(self.Bi[self.item_map[i]])
        
        #Latent factor variables initialized from a uniform distribution
        self.Gu = dict()                      #user latent factor vector repr
        self.Gi = dict()                      #item latent factor vector repr
        
        for i in range(len(self.user_map)):
            self.Gu[self.user_map[i]] = np.random.uniform(0,1,(1,self.k)) 
            flat_theta.extend(np.array(list(self.Gu[self.user_map[i]])).flatten())
        for i in range(len(self.item_map)):
            self.Gi[self.item_map[i]] = np.random.uniform(0,1,(1,self.k)) 
            flat_theta.extend(np.array(list(self.Gi[self.item_map[i]])).flatten())
        
        self.totalparams = 1 + len(self.user_set) + len(self.item_set) +\
                        self.k*(len(self.user_set) + len(self.item_set))
        return np.array(flat_theta)
    
    
    def retrieve_theta_components(self,theta):
        self.alpha = theta[0]
        j=1
        for i in range(len(self.user_set)):
            self.Bu[self.user_map[i]] = theta[j]
            j+=1
        for i in range(len(self.item_set)):
            self.Bi[self.item_map[i]] = theta[j]
            j+=1
        for i in range(len(self.user_set)):
            self.Gu[self.user_map[i]] = np.array(theta[j:j+self.k])
            j += self.k
        for i in range(len(self.item_set)):
            self.Gi[self.item_map[i]] = np.array(theta[j:j+self.k])
            j+=self.k
        #print(j, self.final_param,len(theta))
        if j!= len(theta):
            print("Something went wrong. Not all theta values were used")
        
    def pred(self,user,item):
        ''' calculates the current prediction by the model for the rating of user for the given item'''
        return self.alpha + self.Bu[user] + self.Bi[item]+ np.asscalar(np.dot(self.Gu[user], self.Gi[item].T))
    
    def f(self,theta):
        '''Calculates the value of the objective function (loss) on the training data. 
            Note that the training error is not MSE '''
        #retrieve the individual components of theta
        self.retrieve_theta_components(theta)
        error = 0
        for review in self.train_data:
            user = review['review/userId']
            item = review["product/productId"]
            error += (self.pred(user,item) - review["review/score"])**2
        error/=self.Ntrain
        #regularization terms 
        Bu_np = np.array(list(self.Bu.values()))
        Bi_np = np.array(list(self.Bi.values()))
        reg_complexity = np.sum(np.square(Bu_np)) + np.sum(np.square(Bi_np))
        for user in self.Gu:
            reg_complexity += np.linalg.norm(self.Gu[user])**2
        for item in self.Gi:
            reg_complexity += np.linalg.norm(self.Gi[item])**2
        total_mse = error + self.Lambda * reg_complexity
        return total_mse/2.0
    
    def fprime_one_func(self,theta):
        self.retrieve_theta_components(theta)
        flat_gradient = []
        umap_len = len(self.user_map)
        imap_len = len(self.item_map)
        
        self.alpha_grad = 0                              
        self.Bu_grad = dict()                                      
        self.Bi_grad = dict()                                     
        self.Gu_grad = dict()                      
        self.Gi_grad = dict()   
        for i in range(len(self.user_map)):
            self.Bu_grad[self.user_map[i]] = 0.0
            self.Gu_grad[self.user_map[i]] = np.zeros((1,self.k)) 
        for i in range(len(self.item_map)):
            self.Bi_grad[self.item_map[i]] = 0.0
            self.Gi_grad[self.item_map[i]] = np.zeros((1,self.k))
        
        for review in self.train_data:
            user = review["review/userId"]
            item = review["product/productId"]
            delta = self.pred(user,item) - review["review/score"]
            delta /= self.Ntrain
            self.alpha_grad += delta
            self.Bu_grad[user]+= delta
            self.Bi_grad[item]+= delta
            self.Gu_grad[user]+= delta * self.Gi[item]
            self.Gi_grad[item]+= delta * self.Gu[user]
        
        for user in self.user_set:
            self.Bu_grad[user] += self.Lambda * self.Bu[user]
            self.Gu_grad[user] += self.Lambda * self.Gu[user]
        for item in self.item_set:
            self.Bi_grad[item] += self.Lambda * self.Bi[item]
            self.Gi_grad[item] += self.Lambda * self.Gi[item]
        
        flat_gradient.append(self.alpha_grad)
        for i in range(len(self.user_set)):
            flat_gradient.append(self.Bu_grad[self.user_map[i]])
        for i in range(len(self.item_set)):
            flat_gradient.append(self.Bi_grad[self.item_map[i]])
        for i in range(len(self.user_set)):
            flat_gradient.extend(np.array(self.Gu_grad[self.user_map[i]]).flatten())
        for i in range(len(self.item_set)):
            flat_gradient.extend(np.array(self.Gi_grad[self.item_map[i]]).flatten())
        
        return np.array(flat_gradient)
        
    
    def fprime(self, theta):
        self.retrieve_theta_components(theta)
        flat_gradient = [self.compute_gradient_wrt_alpha()]
        Bu_grad = self.compute_gradient_wrt_Bu()
        Bi_grad = self.compute_gradient_wrt_Bi()
        Gu_grad = self.compute_gradient_wrt_Gu()
        Gi_grad = self.compute_gradient_wrt_Gi()
        for i in range(len(self.user_set)):
            flat_gradient.append(Bu_grad[self.user_map[i]])
        for i in range(len(self.item_set)):
            flat_gradient.append(Bi_grad[self.item_map[i]])
        for i in range(len(self.user_set)):
            flat_gradient.extend(np.array(Gu_grad[self.user_map[i]]).flatten())
        for i in range(len(self.item_set)):
            flat_gradient.extend(np.array(Gi_grad[self.item_map[i]]).flatten())
        return np.array(flat_gradient)
    
    def vanilla_gd(self,eta):
        flat_theta_guess =np.array([0]*self.totalparams, dtype = np.float64)
        for i in range(1000):
            flat_gradient = self.fprime(flat_theta_guess)
            flat_theta_guess -= eta*flat_gradient
            print("{} U : Objective value: {}".format(i,self.f(flat_theta_guess)))
    
    def call(self,theta):
        print("{} Objective value: {}".format(self.i, self.f(theta)))
        self.i+=1
    
    def objectiveloss_lbfgs(self,grad_tolerance,fac):
        self.i =0;
        flat_theta_guess = self.final_param  #start with the initial guess or the one made by previous call to func
        flat_theta,value,d = opt.fmin_l_bfgs_b(self.f,flat_theta_guess,self.fprime_one_func,\
                                               pgtol = grad_tolerance,disp=True,\
                                              callback = self.call, iprint=0)
        print("Value of ojective function {} at pgtol={}".format(value,grad_tolerance))
        self.final_param = flat_theta
    
    def mse_test(self,test_data):
        ''' Uses Mean Squared Error as evaluation metric on test data provided by user'''
        self.retrieve_theta_components(self.final_param)
        error = 0
        unknown_data_count =0;
        for review in test_data:
            if review["review/userId"] in self.Bu and review["product/productId"] in self.Bi:
                user = review["review/userId"]
                item = review["product/productId"]
                error += ( self.pred(user,item) - review["review/score"] )**2
            else: 
                unknown_data_count+=1
        if unknown_data_count>0:
            print("Warning! Unknown {} new data rows".format(unknown_data_count))
        return error / (len(test_data) - unknown_data_count)
    
    def compute_gradient_wrt_alpha(self):
        ''' Compute gradient of objective with respect to alpha parameter'''
        tempsum = 0
        for review in self.train_data:  #each user item id combo
            user = review["review/userId"]
            item = review["product/productId"]
            tempsum += (self.pred(user,item) - review["review/score"])
        return tempsum/self.Ntrain
    
    def compute_gradient_wrt_Bu(self):
        ''' Compute gradient of objective with respect to Bu parameter'''
        Bu_grad = {}
        for user in self.Bu:   
            total = 0.0
            for item in self.Iu[user]:
                total+= (self.pred(user,item) - self.quickmap[user+'-'+item]["review/score"]) 
            total = total/self.Ntrain + self.Lambda*self.Bu[user]
            Bu_grad[user] = total
        return Bu_grad
    
    def compute_gradient_wrt_Bi(self):
        ''' Compute gradient of objective with respect to Bi parameter'''
        Bi_grad = {}
        for item in self.Bi:
            total = 0.0
            for user in self.Ui[item]:
                total+= (self.pred(user,item) - self.quickmap[user+'-'+item]["review/score"]) 
            total = total/self.Ntrain + self.Lambda*self.Bi[item]
            Bi_grad[item] = total
        return Bi_grad
    
    def compute_gradient_wrt_Gu(self):
        ''' Compute gradient of objective with respect to Gu parameter'''
        Gu_grad  = {} 
        for user in self.Gu:   
            total = np.zeros((1,self.k)) 
            for item in self.Iu[user]:
                Rui = self.quickmap[user+'-'+item]["review/score"]
                total+= np.multiply(self.pred(user,item)- Rui, self.Gi[item])
            total = total/self.Ntrain + self.Lambda*self.Gu[user]
            Gu_grad[user] = total.copy()
        #print("Gu grad lenght: {}".format(len(np.array(list(Gu_grad.values())).flatten())))
        return Gu_grad
    
    def compute_gradient_wrt_Gi(self):
        ''' Compute gradient of objective with respect to Gi parameter'''
        Gi_grad = {}
        for item in self.Gi:
            total = np.zeros((1,self.k))
            for user in self.Ui[item]:
                Rui = self.quickmap[user+'-'+item]["review/score"]
                total+= np.multiply(self.pred(user,item)- Rui, self.Gu[user])
            total = total/self.Ntrain + self.Lambda*self.Gi[item]
            Gi_grad[item] = total.copy()
       
        #print("Gi grad lenght: {}".format(len(np.array(list(Gi_grad.values())).flatten())))
        return Gi_grad
    
        
    def deepish_copy(self,org):
        '''
        much, much faster than deepcopy, for a dict of the simple python types.
        '''
        out = dict().fromkeys(org)
        for k,v in org.items():
            try:
                out[k] = v.copy()   # dicts, sets
            except AttributeError:
                try:
                    out[k] = v[:]   # lists, tuples, strings, unicode
                except TypeError:
                    out[k] = v      # ints

        return out  
lfmObj = LFM(train,Iu, Ui,quickmap,distinct_user_set,distinct_item_set,5,1)
grad1 = lfmObj.fprime(lfmObj.final_param)
round(np.mean(np.subtract(grad1,grad2)))
grad2 = lfmObj.fprime_one_func(lfmObj.final_param)
lfmObj.objectiveloss_lbfgs(1e-5,1e2)
finefood_data[0]
lfmObj.mse_test(test)
lfmObj.alpha
np.savetxt("ratebeer_lfm_results", lfmObj.final_param)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11864064602960969, 'fraction_numerical': 0.004643337819650067, 'mean_word_length': 2.139205745669624, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '371034', 'license': 'MIT', 'n_tokens_mistral': 4724, 'n_tokens_neox': 4598, 'n_words': 955, 'path': 'ratebeerstandardLFM-LBFGS.ipynb', 'repo_name': 'KannarKK/ExpertiseEvolutionModel', 'size': '14860'}