<jupyter_start><jupyter_text>Authors:    <NAME> (axa)    <NAME> (bpalmagu) ML-Jokes: Collaborative Filtering   Setup<jupyter_code># Path to ml-jokes folder
import os
if os.getcwd().split('/')[-2] == 'ml-jokes': os.chdir('..')
print(f'Current directory: {os.getcwd()}')
assert set(['data', 'mljokes', 'environment.yml', 'nbs']) <= set(os.listdir()), \
    'Wrong path; go to ./heinz-95729-project/api/ml-jokes'
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from surprise import SVD
from surprise import Dataset, Reader
from surprise import accuracy
from surprise import dump
from surprise.model_selection import cross_validate, train_test_split, GridSearchCV
from mljokes.data import read_jokes, read_ratings, load_test_idx
from mljokes import cf_utils as cfu<jupyter_output><empty_output><jupyter_text>Data Prep<jupyter_code># Read jokes
jokes = read_jokes()
# Read and consolidate jester ratings
test_idx = load_test_idx()
ratings = read_ratings()
ratings['test_idx'] = test_idx

del test_idx

ratings.head(3)
# Filter out missing ratings (99.0)
unseen = ratings.loc[ratings['rating']==99]
ratings = ratings.loc[ratings['rating']!=99]
ratings.reset_index(inplace=True, drop=True)
ratings.sort_values(by=['user_id', 'joke_id'], ascending=True, inplace=True, ignore_index=True)
ratings.head(3)<jupyter_output><empty_output><jupyter_text>SVD Model<jupyter_code># Build custom dataset 
reader = Reader(rating_scale=(ratings.rating.min(), ratings.rating.max()))
data = Dataset.load_from_df(ratings[['user_id', 'joke_id', 'rating']], reader)
# Hyperparameter tuning
param_grid = {'lr_all': [0.002, 0.005], \
    'n_factors': [10, 15, 20, 30],\
        'biased': [True, False],\
            'reg_all': [0.02, 0.05]}

gs = GridSearchCV(SVD, param_grid, measures=['mae', 'rmse'], cv=3)
gs.fit(data)

# Best MAE score
print(gs.best_score['mae'])

# Combination of parameters that gave the best MAE score
print(gs.best_params['mae'])
# Train model with full train dataset
data = Dataset.load_from_df(ratings.loc[ratings['test_idx']==0,['user_id', 'joke_id', 'rating']], reader)
trainset = data.build_full_trainset()

model = gs.best_estimator['mae']
model.fit(trainset)

# Get MAE on test set
test_data = Dataset.load_from_df(ratings.loc[ratings['test_idx']==1,['user_id', 'joke_id', 'rating']], reader)
_, test_data = train_test_split(test_data, test_size=1.0)

predictions = model.test(test_data)
accuracy.mae(predictions)

# Save model
file_name = './results/cf_bestModel_mae'
dump.dump(file_name, algo=model, verbose=0)<jupyter_output><empty_output><jupyter_text>Get all predictions<jupyter_code>all_ratings = cfu.get_all_ratings(model)
all_ratings.to_pickle('./results/cf_preds.pkl')<jupyter_output><empty_output><jupyter_text>Get Recommendations<jupyter_code># Load model
file_name = './results/cf_bestModel_mae'
_, model = dump.load(file_name)
# Choose user
user_id = 4493
unseen_by_user = unseen.values[unseen['user_id']==user_id, 1]
cfu.get_user_profile(model, ratings, jokes, user_id, 3, 10, unseen_by_user)
# Visualize joke-concept matrix with predicted ratings from selected user
joke_embeddings = cfu.generate_joke_embeddings(model, perplexity = 6, random_state = 3)
cfu.plot_user_preferences(model, joke_embeddings, user_id,unseen_by_user, n_recommended=3, mask_unseen=False)<jupyter_output>/home/brendapalmag/anaconda3/envs/ecommerce_jokes/lib/python3.8/site-packages/sklearn/manifold/_t_sne.py:982: FutureWarning: The PCA initialization in TSNE will change to have the standard deviation of PC1 equal to 1e-4 in 1.2. This will ensure better convergence.
  warnings.warn(
{'fraction_non_alphanumeric': 0.13607331296861983, 'fraction_numerical': 0.018050541516245487, 'mean_word_length': 6.847494553376906, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '428689', 'license': 'MIT', 'n_tokens_mistral': 1373, 'n_tokens_neox': 1311, 'n_words': 276, 'path': 'api/ml-jokes/nbs/collaborative_filtering.ipynb', 'repo_name': 'alejandroxag/heinz-95729-project', 'size': '3618'}