<jupyter_start><jupyter_text>This notebook (Update 12.14)* Try three different collaborative filterings: Memory, SVD, NMF* Explore SVD performance with different n_components<jupyter_code>import pandas as pd
import numpy as np
from scipy.sparse import csr_matrix
from scipy.sparse import coo_matrix, vstack
import pickle<jupyter_output><empty_output><jupyter_text>Load the data and convert into sparse matrix<jupyter_code>def read_data(file_path):
    from scipy.sparse import csr_matrix
    
    # Read pickle
    stu = pd.read_pickle(file_path)
    
    # Num of dictionaries in the file
    n = len(stu)
    
    # Initialize df
    df = pd.DataFrame()
    
    # Loop through dicts and concat
    for i in range(n):
        stu[i]['student_id'] = i
        df = pd.concat([df,stu[i]])
    
    # df = user dataset we have
    # user_input: courses that user provide

    # Fix .0 issues -- Some courses has .0 at the end (e.g. AAS 101.0), remove .0
    df.loc[df['Subject/Catalog'].str.contains('\.') == True, 'Subject/Catalog'] = df.loc[df['Subject/Catalog'].str.contains('\.') == True, 'Subject/Catalog'].str.split('\.').str[0]
    
    # Pivot table to stu - course
    df = pd.pivot_table(df, values = 'Rating', index = 'student_id', columns = 'Subject/Catalog')
    
    # Fillna
    df = df.fillna(0)
    
    # df to sparse matrix
    csr = csr_matrix(df)
    
    # Store course names
    courses = list(df.columns)
    

    return csr, courses 
#csr, courses = read_data('user_profiles.pickle')<jupyter_output><empty_output><jupyter_text>Store user profile csr and the list of courses<jupyter_code>with open('csr.pkl', "wb") as fOut:
    pickle.dump(csr, fOut, protocol=pickle.HIGHEST_PROTOCOL)
    
with open('courses.pkl', "wb") as fOut:
    pickle.dump(courses, fOut, protocol=pickle.HIGHEST_PROTOCOL)<jupyter_output><empty_output><jupyter_text>Load csr and courses<jupyter_code>with open('csr.pkl', "rb") as fIn:
    csr = pickle.load(fIn)
with open('courses.pkl', "rb") as fIn:
     courses = pickle.load(fIn)
# random user input
user_input = np.random.choice(courses, size = 10, replace = False)<jupyter_output><empty_output><jupyter_text>Matrix-based collaborative filtering<jupyter_code>from sklearn.decomposition import TruncatedSVD

def svd_based(csr, courses, user_input, num_factors = 100, top_n = 10, random_state=42):
    # SVD based collaborative filtering
    user_row = np.isin(courses, user_input)
    uc_mat = vstack([csr, user_row]).tocsr()
    
    user_idx = -1

    
    # SVD
    svd = TruncatedSVD(n_components = num_factors, random_state = random_state)
    
    # Transform X
    X_transformed = svd.fit_transform(uc_mat)
    
    # Calculate r
    r = np.matmul(X_transformed, svd.components_)
    
    # Pred rating
    pred_rating = r[user_idx,:]#.reshape(1,-1)
    
    
    # Matched user vector
    user_vec = uc_mat[user_idx].toarray().flatten()
    
    
    # User's favorite course indices
    user_fav_idx = np.argsort(-user_vec)
    
    # User's favorite courses
    #user_fav = [courses[i] for i in user_fav_idx[:top_n]]
    #print(f"Top {top_n} favorite courses:")
    #print(user_fav)

    
    # Flatten pred_rating
    pred_rating = np.asarray(pred_rating).flatten()
    
    # Argsort the rating except the courses taken by the user
    recommend_index = np.argsort(-pred_rating[user_vec == 0])
    
    # Recommend top courses
    top_courses = [courses[i] for i in recommend_index[:top_n]]

    # RMSE between true & predicted. Uncomment the next two lines to see RMSE
    rmse = np.sqrt(np.sum((user_vec[user_vec != 0] - pred_rating[user_vec != 0]) ** 2))
    #print(f"RMSE = {rmse}")
    
    return top_courses, rmse<jupyter_output><empty_output><jupyter_text>-----------------  RMSE & Runtime<jupyter_code>r_list = []
for _ in range(100):
    user_input = np.random.choice(courses, size = 10, replace = False)
    pred, rmse = svd_based(csr, courses, user_input, num_factors = 200)
    r_list.append(rmse)
print(np.average(r_list))
%%timeit
user_input = np.random.choice(courses, size = 10, replace = False)
pred, rmse = svd_based(csr, courses, user_input, num_factors = 200)
subcat = df['Subject/Catalog'].unique()
import numpy as np
def svd_tune(df, num_factors, top_n = 10, random_state=42):
    # SVD based collaborative filtering
    
    # user_idx = -1
    user_idx = -1
    rmse_list = []
    for i in range(100):
        user_input = np.random.choice(courses, size = 10, replace = False)
        uc_mat, courses = concat_user_input_and_convert_to_csr(df, user_input)

        # SVD
        svd = TruncatedSVD(n_components = num_factors, random_state = random_state)

        # Transform X
        X_transformed = svd.fit_transform(uc_mat)

        # Calculate r
        r = np.matmul(X_transformed, svd.components_)

        # Pred rating
        pred_rating = r[user_idx,:]#.reshape(1,-1)


        # Matched user vector
        user_vec = uc_mat[user_idx].toarray().flatten()


        # User's favorite course indices
        user_fav_idx = np.argsort(-user_vec)

        # User's favorite courses
        #user_fav = [courses[i] for i in user_fav_idx[:top_n]]
        #print(f"Top {top_n} favorite courses:")
        #print(user_fav)


        # Flatten pred_rating
        pred_rating = np.asarray(pred_rating).flatten()

        # Argsort the rating except the courses taken by the user
        recommend_index = np.argsort(-pred_rating[user_vec == 0])

        # Recommend top courses
        top_courses = [courses[i] for i in recommend_index[:top_n]]


        # RMSE between true & predicted. Uncomment the next two lines to see RMSE
        rmse_list.append(np.sqrt(np.sum((user_vec[user_vec != 0] - pred_rating[user_vec != 0]) ** 2)))
    avg_rmse = np.average(rmse_list)
    #print(f"RMSE = {avg_rmse}")
    
    return avg_rmse<jupyter_output><empty_output><jupyter_text>------------- SVD runtime by n_components<jupyter_code>def svd_time(csr, num_factors, top_n = 10, random_state=42):
    # SVD based collaborative filtering
    
    # user_idx = -1
    user_idx = -1
    rmse_list = []
    for i in range(10):
        user_input = np.random.choice(courses, size = 10, replace = False)

        user_row = np.isin(courses, user_input)
        uc_mat = vstack([csr, user_row]).tocsr()

        user_idx = -1

        # SVD
        svd = TruncatedSVD(n_components = num_factors, random_state = random_state)

        # Transform X
        X_transformed = svd.fit_transform(uc_mat)

        # Calculate r
        r = np.matmul(X_transformed, svd.components_)

        # Pred rating
        pred_rating = r[user_idx,:]#.reshape(1,-1)


        # Matched user vector
        user_vec = uc_mat[user_idx].toarray().flatten()


        # User's favorite course indices
        user_fav_idx = np.argsort(-user_vec)

        # User's favorite courses
        #user_fav = [courses[i] for i in user_fav_idx[:top_n]]
        #print(f"Top {top_n} favorite courses:")
        #print(user_fav)


        # Flatten pred_rating
        pred_rating = np.asarray(pred_rating).flatten()

        # Argsort the rating except the courses taken by the user
        recommend_index = np.argsort(-pred_rating[user_vec == 0])

        # Recommend top courses
        top_courses = [courses[i] for i in recommend_index[:top_n]]

    
    return None
%%timeit
svd_based(csr, courses, user_input, num_factors = 20)
%%timeit
svd_based(csr, courses, user_input, num_factors = 60)
%%timeit
svd_based(csr, courses, user_input, num_factors = 100)
%%timeit
svd_based(csr, courses, user_input, num_factors = 140)
%%timeit
svd_based(csr, courses, user_input, num_factors = 180)
%%timeit
svd_based(csr, courses, user_input, num_factors = 220)
%%timeit
svd_based(csr, courses, user_input, num_factors = 260)
%%timeit
svd_based(csr, courses, user_input, num_factors = 300)<jupyter_output>2.24 s ± 316 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)<jupyter_text>Plot - time by n_components<jupyter_code>import matplotlib.pyplot as plt
import seaborn as sns

x = np.arange(20, 340, 40)
runtime_list = [0.258, 0.504, 0.574, 0.797, 1.11, 1.25, 1.7, 2.24]

fig, ax1 = plt.subplots()

ax1.plot(x, runtime_list, color = 'green')
ax1.set_ylabel('Runtime')
ax1.set_xlabel('n_components')
ax1.axhline(1, linestyle = '--', color = 'grey')
ax1.set_title("Runtime by n_components - SVD")
plt.show()<jupyter_output><empty_output><jupyter_text>Explained Variance<jupyter_code>import numpy as np
def svd_var(csr, num_factors, top_n = 10, random_state=42):
    # SVD based collaborative filtering
    
    # user_idx = -1
    user_idx = -1
    explained_variance_list = []
    rmse_list = []

    for i in range(100):
        user_input = np.random.choice(courses, size = 10, replace = False)

        user_row = np.isin(courses, user_input)
        uc_mat = vstack([csr, user_row]).tocsr()


        # SVD
        svd = TruncatedSVD(n_components = num_factors, random_state = random_state)

        # Transform X
        X_transformed = svd.fit_transform(uc_mat)

        # Calculate r
        r = np.matmul(X_transformed, svd.components_)

        # Pred rating
        pred_rating = r[user_idx,:]#.reshape(1,-1)


        # Matched user vector
        user_vec = uc_mat[user_idx].toarray().flatten()


        # User's favorite course indices
        user_fav_idx = np.argsort(-user_vec)

        # User's favorite courses
        #user_fav = [courses[i] for i in user_fav_idx[:top_n]]
        #print(f"Top {top_n} favorite courses:")
        #print(user_fav)


        # Flatten pred_rating
        pred_rating = np.asarray(pred_rating).flatten()

        # Argsort the rating except the courses taken by the user
        recommend_index = np.argsort(-pred_rating[user_vec == 0])

        # Recommend top courses
        top_courses = [courses[i] for i in recommend_index[:top_n]]


        # RMSE between true & predicted. 
        rmse_list.append(np.sqrt(np.sum((user_vec[user_vec != 0] - pred_rating[user_vec != 0]) ** 2)))
        
        # Explained variance 
        explained_variance_list.append(svd.explained_variance_ratio_.sum())
    
    avg_ev = np.average(explained_variance_list)
    avg_rmse = np.average(rmse_list)

    
    return avg_ev, avg_rmse
n = np.arange(50, 500, 50)
avg_ev_list = []
avg_rmse_list = []
for num in n:
    avg_ev, avg_rmse = svd_var(df, num, top_n = 10, random_state=42)
    avg_ev_list.append(avg_ev)
    avg_rmse_list.append(avg_rmse)
    #print("n_components: ",num, " Explained variance sum: ", avg_ev, ' RMSE: ', avg_rmse)
    
n = np.arange(500, 800, 50)
for num in n:
    avg_ev, avg_rmse = svd_var(df, num, top_n = 10, random_state=42)
    avg_ev_list.append(avg_ev)
    avg_rmse_list.append(avg_rmse)
    #print("n_components: ",num, " Explained variance sum: ", avg_ev, ' RMSE: ', avg_rmse)<jupyter_output>n_components:  500  Explained variance sum:  0.6187698018564972  RMSE:  2.8370689935019273
n_components:  550  Explained variance sum:  0.642819555062698  RMSE:  2.756939016177228
n_components:  600  Explained variance sum:  0.6654379541743018  RMSE:  2.7701917618079115
n_components:  650  Explained variance sum:  0.6865863661990289  RMSE:  2.6958564337227897
n_components:  700  Explained variance sum:  0.706502879828708  RMSE:  2.7041602167885803
n_components:  750  Explained variance sum:  0.7252492577436896  RMSE:  2.6473528552518335<jupyter_text>Explained variance goes up and RMSE goes down as n_components gets higher.<jupyter_code>import matplotlib.pyplot as plt
import seaborn as sns
fig, ax1 = plt.subplots()

x = np.arange(50, 800, 50)
ax1.plot(x, avg_ev_list, color = 'green')
ax1.set_ylabel('Explained Variance ratio sum', color = 'green')
ax2 = ax1.twinx()
ax2.plot(x, avg_rmse_list)
ax2.set_ylabel('RMSE', color = 'blue')
plt.show()
n = np.arange(20, 320, 20)

avg_rmse_list = []
avg_ev_list = []
for num in n:
    avg_ev, avg_rmse = svd_var(csr, num, top_n = 10, random_state=42)
    avg_ev_list.append(avg_ev)
    avg_rmse_list.append(avg_rmse)
    #print("n_components: ",num, " Explained variance sum: ", avg_ev, ' RMSE: ', avg_rmse)
    
#avg_rmse_list, avg_ev_list
n = np.arange(20, 320, 20)
fig, ax1 = plt.subplots()
color1 = '#B42E40'
color2 = '#532EB4'
ax1.plot(n, avg_ev_list, color = color1)
ax1.set_ylabel('Sum of Explained Variance Ratio', color = color1)
ax2 = ax1.twinx()
ax1.axvline(200, linestyle = '--', color = 'grey')
ax2.plot(n, avg_rmse_list, color = color2)
ax2.set_ylabel('RMSE', color = color2)
ax1.set_xlabel('n_components')
ax1.set_title('Sum of Explained Variance Ratio\nand RMSE by n_components - SVD')
plt.show()<jupyter_output><empty_output><jupyter_text>---------- Memory-based collaborative filtering<jupyter_code>import scipy.sparse
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity


def memory_based(csr, courses, user_input, top_n):
    # Memory-based collaborative filtering
    user_idx = -1
    
    user_row = np.isin(courses, user_input)
    uc_mat = vstack([csr, user_row]).tocsr()
    # Normalize user-course matrix
    norm_uc_mat = uc_mat - uc_mat.mean(axis = 1).reshape(-1,1)
    
    # Cosine similarity of raw matrix
    cossim = cosine_similarity(uc_mat)
    
    # Calculate the numerator of V-hat(aj)
    v_num = np.dot(cossim[user_idx,:], norm_uc_mat)
    
    # Sum of V-hat(aj) numerator
    v_num_sum = v_num.sum(axis = 0)
    
    # V_hat(aj)
    v_hat = v_num_sum/(cossim[user_idx,:].sum())
    
    #Predicted rating of each course
    pred_rating = np.add(v_hat, uc_mat.mean(axis = 1).reshape(-1,1)[user_idx])
    
    # User vector
    user_vec = uc_mat[user_idx].toarray().flatten()
    
    # User's favorite course indices
    user_fav_idx = np.argsort(-user_vec)
    
    # User's favorite courses
    user_fav = [courses[i] for i in user_fav_idx[:top_n]]
    #print(f"Top {top_n} favorite courses:")
    #print(user_fav)
    
    # Flatten pred_rating
    pred_rating = np.asarray(pred_rating).flatten()
    
    
    # Argsort the rating except the courses taken by the user
    recommend_index = np.argsort(-pred_rating[user_vec == 0])
    
    # Recommend top courses
    top_courses = [courses[i] for i in recommend_index[:top_n]]
    
    #print(f"Top {top_n} recommended courses:")
    #print(top_courses)

    #print()

    # RMSE between true & predicted
    rmse = np.sqrt(np.sum((user_vec[user_vec != 0] - pred_rating[user_vec != 0]) ** 2))
    #print(f"RMSE = {rmse}")
    
    return top_courses, rmse


%%timeit
user_input = np.random.choice(courses, size = 10, replace = False)
pred = memory_based(csr, courses, user_input, 10)
r_list = []

for _ in range(100):
    user_input = np.random.choice(courses, size = 10, replace = False)
    pred, rmse = memory_based(csr, courses, user_input, 10)
    r_list.append(rmse)
print(np.average(r_list))<jupyter_output>2.3113030911012737<jupyter_text>NMF-based collaborative filtering<jupyter_code>from sklearn.decomposition import NMF

def nmf_based(uc_mat, courses, user_input, num_factors = 80, top_n = 10, random_state=42):
    # NMF based collaborative filtering
    user_row = np.isin(courses, user_input)
    uc_mat = vstack([csr, user_row]).tocsr()    
    # NMF
    nmf = NMF(n_components = num_factors, init = 'nndsvd', max_iter = 300, random_state = random_state)
    
    # Transform X
    X_transformed = nmf.fit_transform(uc_mat)
    
    # Calculate r
    r = np.matmul(X_transformed, nmf.components_)
    
    # Predict rating
    pred_rating = r[user_idx,:].reshape(1,-1)
    
    
    # User vector
    user_vec = uc_mat[user_idx].toarray().flatten()
    
    # User's favorite course indices
    user_fav_idx = np.argsort(-user_vec)
    
    # User's favorite courses
    user_fav = [courses[i] for i in user_fav_idx[:top_n]]
    #print(f"Top {top_n} favorite courses:")
    #print(user_fav)
    
    # Flatten pred_rating
    pred_rating = np.asarray(pred_rating).flatten()
    
    
    # Argsort the rating except the courses taken by the user
    recommend_index = np.argsort(-pred_rating[user_vec == 0])
    
    # Recommend top courses
    top_courses = [courses[i] for i in recommend_index[:top_n]]
    
    #print(f"Top {top_n} recommended courses:")
    #print(top_courses)

    #print()

    # RMSE between true & predicted
    rmse = np.sqrt(np.sum((user_vec[user_vec != 0] - pred_rating[user_vec != 0]) ** 2))
    #print(f"RMSE = {rmse}")
    
    return top_courses, rmse


#%%timeit
#pred = nmf_based(csr, courses, user_input, 80, )
r_list = []

for _ in range(10):
    user_input = np.random.choice(courses, size = 10, replace = False)
    pred, rmse = nmf_based(csr, courses, user_input, 10)
    r_list.append(rmse)
print(np.average(r_list))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11870201535508637, 'fraction_numerical': 0.03148992322456814, 'mean_word_length': 3.3726724364017833, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '423672', 'license': 'CC0-1.0', 'n_tokens_mistral': 6616, 'n_tokens_neox': 6039, 'n_words': 1337, 'path': 'Exploration/collab_filtering_exploration.ipynb', 'repo_name': 'beileihe/697-Capstone', 'size': '16672'}