<jupyter_start><jupyter_text>Neural Collaborative Filtering---<jupyter_code>import pandas as pd
import numpy as np

# sklearn imports
from sklearn.model_selection import train_test_split
# visualization imports
import matplotlib.pyplot as plt
import sys
sys.path.append('../src')
%load_ext autoreload
%autoreload 2
df_ratings = pd.read_csv('../Datasets/dataset/ydata-ymusic-user-artist-ratings-v1_0.txt', sep='\t',header=None, encoding="ISO-8859-1", names=['user_id', 'artist_id','ratings'])
df_ratings.replace(0, np.nan, inplace=True)
df_ratings.replace(255, 0, inplace=True)
df_ratings.dropna(inplace=True)
num_users = df_ratings.user_id.nunique()
num_items = df_ratings.artist_id.nunique()
print('There are {} unique users and {} unique movies in this data set'.format(num_users, num_items))
user_maxId = df_ratings.user_id.max()
item_maxId = df_ratings.artist_id.max()
print('There are {} distinct users and the max of user ID is also {}'.format(num_users, user_maxId))
print('There are {} distinct movies, however, the max of artist ID is {}'.format(num_items, item_maxId))<jupyter_output>There are 1940250 distinct users and the max of user ID is also 1948882
There are 50080 distinct movies, however, the max of artist ID is 1101719<jupyter_text>Pour des raisons de dimensions, on réindexe les ID des artistes<jupyter_code>def reduce_item_dim(df_ratings):
    """
    Reduce item vector dimension to the number of distinct items in our data sets
    
    input: pd.DataFrame, df_ratings should have columns ['user_id', 'artist_id', 'ratings']
    output: pd.DataFrame, df_ratings with new 'artist_id' that is compressed
    """
    # pivot
    df_user_item = df_ratings.pivot(index='user_id', columns='artist_id', values='ratings')
    # reset movieId
    df_user_item = df_user_item.T.reset_index(drop=True).T
    # undo pivot/melt - compress data frame
    df_ratings_new = df_user_item \
        .reset_index('user_id') \
        .melt(
            id_vars='user_id', 
            value_vars=df_user_item.columns,
            var_name='artist_id',
            value_name='ratings')
    # drop nan and final clean up
    return df_ratings_new.dropna().sort_values(['user_id', 'artist_id']).reset_index(drop=True)
df_ratings.head()
df_ratings.artist_id = df_ratings.artist_id.astype('category').cat.codes
df_ratings.artist_id += 1
df_ratings.head()
df_train, df_test = train_test_split(df_ratings.head(10000), test_size=0.2, shuffle=True, random_state=99)
print('shape of training data set:')
print(df_train.shape)
print('shape of test data set:')
print(df_test.shape)
from NeuMF import NeuMF

NCF = NeuMF(num_users=num_users,
    num_items=num_items,
    MF_dim=10,
    MF_reg=(0, 0),
    MLP_layers=[64, 32, 16, 8],
    MLP_regs=[0, 0, 0, 0])
model_ncf = NCF.build_model()
model_ncf.summary()
from NeuMF import Trainer

# model config
BATCH_SIZE = 64
EPOCHS = 10
VAL_SPLIT = 0.35

trainer = Trainer('adam', BATCH_SIZE, EPOCHS, VAL_SPLIT, inputs=[df_train.user_id.values, df_train.artist_id.values], outputs=df_train.ratings.values)
                      
history = trainer.train_model(model_ncf)
trainer.plot_learning_curve(history, 'rmse')
# define rmse function
rmse = lambda true, pred: np.sqrt(np.mean(np.square(np.squeeze(predictions) - np.squeeze(true))))

model_ncf_trained = trainer.load_trained_model(model_ncf, 'models/tmp/model.hdf5')

# make prediction using test data
predictions = model_ncf_trained.predict([df_test.user_id.values, df_test.artist_id.values])
# get the RMSE
error = rmse(df_test.ratings.values, predictions)
print('The out-of-sample RMSE of rating predictions is', round(error, 4))<jupyter_output>The out-of-sample RMSE of rating predictions is 21.7377
{'fraction_non_alphanumeric': 0.11646803900325027, 'fraction_numerical': 0.021397616468039005, 'mean_word_length': 5.014657980456026, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '194919', 'license': 'MIT', 'n_tokens_mistral': 1384, 'n_tokens_neox': 1307, 'n_words': 300, 'path': 'notebooks/NeuCF.ipynb', 'repo_name': 'EvolveArt/rec_sys_workshop', 'size': '3692'}