<jupyter_start><jupyter_text>TASK 1 PART 1: NON NEGATIVE MATRIX FACTORIZATIONCreating user business matrix and applying a variation of collaborative filtering method where the errors between predictions and true value is minimized using gradient descentReference: http://www.quuxlabs.com/blog/2010/09/matrix-factorization-a-simple-tutorial-and-implementation-in-python/<jupyter_code>#imports go here
import json
import pandas as pd
import numpy as np
import copy
import random
import pickle
import math
import matplotlib.pyplot as plt

from matplotlib.axes import Axes
from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>Data stored in nv_restaurants_reviews.csv consits of reviews for NV based restaurantsNote: Place the nv_restaurants_reviews.csv in the same folder or enter the path<jupyter_code>nevada_pickle=pd.read_csv("nv_restaurants_reviews.csv")
review_df_restaurant = nevada_pickle.loc[:, ['business_id', 'user_id','stars']]
len(review_df_restaurant)<jupyter_output><empty_output><jupyter_text>Averaging the ratings at a user business level<jupyter_code>review_df_restaurant_avg=review_df_restaurant.groupby(['business_id', 'user_id'], as_index=False).agg({"stars": "mean"})
len(review_df_restaurant_avg)
review_df_restaurant_avg.head()<jupyter_output><empty_output><jupyter_text>Sparsity Reduction: Using restaurants with reviews > 50 and users who gave reviews for minimum 20 restaurants<jupyter_code>BusinessGreaterThan50Reviews = review_df_restaurant_avg.groupby('business_id').filter(lambda x: x['business_id'].count()>50)


BusinessGreaterThan50Reviews.business_id.nunique()
BusinessGreaterThan50Reviews.head()
UsersGreaterThan20Reviews = BusinessGreaterThan50Reviews.groupby('user_id').filter(lambda x: x['user_id'].count()> 20)
UsersGreaterThan20Reviews.user_id.nunique()
len(UsersGreaterThan20Reviews)<jupyter_output><empty_output><jupyter_text>Creating user business matrix:<jupyter_code>matrix = UsersGreaterThan20Reviews.pivot(index='user_id', columns='business_id', values='stars').fillna(0)
matrix.shape<jupyter_output><empty_output><jupyter_text>TEST SPLIT:We randomly chose 20 percent of the values in the populated matrix and replaced them with zeros in the matrix. While sampling this 20 percent cells, we made sure of the following:- Corresponding users have rated atleast one other business- Corresponding restaurants are rated by atleast one other user<jupyter_code>#Just to get an idea about the train test split that is supposed to be done
zero_removal = int(0.2 * np.count_nonzero(matrix))
print(np.count_nonzero(matrix),zero_removal)
#Sampling of test and giving them a value of 0
#matrix = [[5,0,0,0],[3,1,2,0],[0,0,3,5],[3,3,1,2],[3,1,2,3]]
array_matrix = np.array(matrix)

row_zero = np.count_nonzero(array_matrix, axis=1)
col_zero = np.count_nonzero(array_matrix, axis=0)
row_nums = dict(enumerate(row_zero))
col_nums = dict(enumerate(col_zero))


#print("Initial Matrix")
#print(array_matrix)
#print(col_nums.keys())
# Get the non-zero samples and make them 0's in train matrix for testing!

import sys
sys.setrecursionlimit(20000)
#LOOP THIS
sample_array = []
sample_count = 0
new_matrix = copy.deepcopy(array_matrix)
non_zero_list = np.nonzero(array_matrix)
non_zero_samples_to_pick = []

for i in range(len(non_zero_list[0])):
    non_zero_samples_to_pick.append([non_zero_list[0][i],non_zero_list[1][i]]) 

def get_sample(non_zero_samples_to_pick,row_nums,col_nums): 
    if len(non_zero_samples_to_pick) <= 0:
        return None
    rand_sample = random.sample(non_zero_samples_to_pick, k=1)
    
    i_val = rand_sample[0][0]
    j_val = rand_sample[0][1]
    
    if(row_nums[i_val]) > 1 and (col_nums[j_val]) > 1:
        
        #sample_count += 1
        sample_array.append([i_val,j_val])
        
        new_matrix[i_val][j_val] = 0
        row_nums[i_val]-=1
        col_nums[j_val]-=1
            
    non_zero_samples_to_pick.remove([i_val,j_val])
    
    #Modify the sample_count to be one third or so.. depending on train test split

for i in range(zero_removal):

    get_sample(non_zero_samples_to_pick,row_nums,col_nums)


user_business_id_test = []
for x,y in sample_array:
    user_business_id_test.append((matrix.index.values[x],matrix.columns[y]))

len(user_business_id_test)<jupyter_output><empty_output><jupyter_text>Same data points for test was used across task 1 both parts<jupyter_code>'''
import pickle
with open ("user_business_id_test.pkl",'wb') as f:
    pickle.dump(user_business_id_test,f)
'''
'''
with open ("user_business_id_test.pkl",'rb') as f:
    test= pickle.load(f)
'''<jupyter_output><empty_output><jupyter_text>Matrix Factorization Implementation- Creating P and Q matrix where K is a tuning parameter- RMSE is calculated for different K's while training<jupyter_code>min_error = float("Inf")
R = np.array(new_matrix)
N = len(R)
M = len(R[0])
def matrix_factorization(R, P, Q, K, alpha,beta,steps = 25):

    Q = Q.T
    for step in range(steps):
        print("Iteration:",step)
        for i,j in R:
            eij = new_matrix[i][j] - np.dot(P[i,:],Q[:,j])
            #print(eij)
                #np.array(P)
            P[i,:] += alpha*(2*eij*Q[:,j])
            P[i,:] -= alpha*beta*P[i,:]
            Q[:,j] += alpha*(2*eij*P[i,:])
            Q[:,j] -= alpha*beta*Q[:,j]

        #eR = np.dot(P,Q)
        e = 0
        for i,j in R:
            e = e + pow(new_matrix[i][j] - np.dot(P[i,:],Q[:,j]), 2)
            
            e = np.sum( (beta/2) * ( np.square(P[i,:]) + np.square(Q[:,j]) ) )
        if e < 0.001:
            break
    nP = P
    nQ = Q.T
    nR = np.dot(nP,nQ.T)

    train_count = np.count_nonzero(R)
    sq_train_error = 0

    for i,j in R:
        sq_train_error += np.square(new_matrix[i][j]-nR[i][j])

    mean_sq_train_error = sq_train_error/train_count
    
    return mean_sq_train_error,nR


after_non_zero_list = np.nonzero(new_matrix)
after_non_zero_samples_to_pick = []

for i in range(len(after_non_zero_list[0])):
    after_non_zero_samples_to_pick.append([after_non_zero_list[0][i],after_non_zero_list[1][i]]) 

#Change the K array accordingly
min_error = float("Inf")
K_array = [100,120,150,180]
final_K_val = K_array[0]
result_array = []

for K in K_array:
    print("K Value : ", K)
    P = np.random.rand(N,K)
    Q = np.random.rand(M,K)
    returned_error, nR_value = matrix_factorization(after_non_zero_samples_to_pick, P, Q, K, .002,.2,steps =30 )
    result_array.append([K,returned_error,nR_value])
    if returned_error < min_error:
        print(returned_error)
        final_K_val = K
        min_error = returned_error
        final_matrix = nR_value
test_error=[]

counter = 0
for mat in result_array:
    mat_error = 0
    
    for i in sample_array:
        mat_error += math.pow((array_matrix[i[0]][i[1]]-mat[2][i[0]][i[1]]),2)
    rmse_mat = math.sqrt(mat_error/len(sample_array))
    test_error.append((K_array[counter],rmse_mat))
    counter+=1


print(test_error)
train_error = [(x[0],x[1]) for x in result_array]
print(train_error)
test_error.sort(key=lambda x:x[0])
train_error.sort(key=lambda x:x[0])
train_errors_sorted = [r[1] for r in train_error]
test_errors_sorted = [r[1] for r in test_error]
k_vals_array = [r[0] for r in train_error]
print(train_error)
print(test_error)<jupyter_output>[(100, 1.1153148977361766), (120, 1.1556476997783187), (150, 1.2309959518458604), (180, 1.316463030257482)]<jupyter_text>Plotting Graph with RMSE<jupyter_code>fig, ax = plt.subplots(figsize = (6,6))
ax.plot(k_vals_array, train_errors_sorted, color='green', label='Training Error')
ax.plot(k_vals_array, test_errors_sorted, color='orange', label='Testing Error')
                                   # <--

ax.set_xlabel('K Value')
ax.set_ylabel('RMSE')
ax.set_title('Train and Test error with K')
plt.legend()
plt.show()

fig, ax = plt.subplots(figsize = (6,6))
ax.plot(k_vals_array, train_errors_sorted, color='green', label='Training Error')


ax.set_xlabel('K Value')
ax.set_ylabel('RMSE')
ax.set_title('Train and Test error with K')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.126143952510512, 'fraction_numerical': 0.027083848627256987, 'mean_word_length': 4.8986141502552885, 'pattern_counts': {'":': 1, '<': 40, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '403132', 'license': 'MIT', 'n_tokens_mistral': 3145, 'n_tokens_neox': 2965, 'n_words': 600, 'path': 'Task1_NMF/Task1_Part1_NMF.ipynb', 'repo_name': 'Dhr11/Yelp_Challenge', 'size': '8086'}