<jupyter_start><jupyter_text>RecommenderThis version is based on a kaggle version   Import<jupyter_code># Ignore  the warnings
import warnings
warnings.filterwarnings('always')
warnings.filterwarnings('ignore')

# data visualisation and manipulation
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import style
#import seaborn as sns
 
#configure
# sets matplotlib to inline and displays graphs below the corressponding cell.
%matplotlib inline  
style.use('fivethirtyeight')
# sns.set(style='whitegrid',color_codes=True)

#model selection
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.metrics import accuracy_score,precision_score,recall_score,confusion_matrix,roc_curve,roc_auc_score
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import LabelEncoder

#preprocess.

from keras.preprocessing.image import ImageDataGenerator

#dl libraraies
import keras
from keras import backend as K
from keras.models import Sequential
from keras.layers import Dense , merge
from keras.optimizers import Adam,SGD,Adagrad,Adadelta,RMSprop
from keras.utils import to_categorical
from keras.utils.vis_utils import model_to_dot
from keras.callbacks import ReduceLROnPlateau


from keras.layers.merge import dot
from keras.models import Model


# specifically for deeplearning.
from keras.layers import Dropout, Flatten,Activation,Input,Embedding
from keras.layers import Conv2D, MaxPooling2D, BatchNormalization
import tensorflow as tf
import random as rn
from IPython.display import SVG
 
# specifically for manipulating zipped images and getting numpy arrays of pixel values of images.
              
import numpy as np  

import os                   
from random import shuffle  
from zipfile import ZipFile



#TL pecific modules
from keras.applications.vgg16 import VGG16<jupyter_output><empty_output><jupyter_text>Exploratory<jupyter_code>column_list = ['userId', 'movieId', 'rating', 'timestamp']
data_content =  pd.read_csv('ml-100k/u.data', delimiter='\t', names=column_list) 

df = data_content.copy()
df.head()
len(df['userId'].unique())
len(df['movieId'].unique())<jupyter_output><empty_output><jupyter_text>We have 943 different users and 1682 different movies. We have 100000 ratings. We have 6.3% ratings. What are the minimum and maximum ratings.<jupyter_code>print(df['rating'].min())
print(df['rating'].max())
df.userId = df.userId.astype('category').cat.codes.values
df.movieId = df.movieId.astype('category').cat.codes.values

# I don't know if it is useful after
df.userId
df['userId'].value_counts(ascending=True)<jupyter_output><empty_output><jupyter_text>Utility matrixThe utility matrix is the matrix where the user are the rows and the movies are the columns. The ratings are the matrix elements.<jupyter_code>index = list(df['userId'].unique())
columns = list(df['movieId'].unique())
index = sorted(index)
columns = sorted(index)

util_df = pd.pivot_table(data=df, values='rating', index ='userId', columns='movieId')
util_df<jupyter_output><empty_output><jupyter_text>The objective is to fill the Nan Value with estimated ratings. First all the NaN are transformed in zeros.<jupyter_code>util_df.fillna(0)<jupyter_output><empty_output><jupyter_text>Training and Validation sets<jupyter_code>users = df.userId.unique()
movies = df.movieId.unique()

userid2idx = {o:i for i,o in enumerate(users)}
movieid2idx = {o:i for i,o in enumerate(movies)}

df['userId'] = df['userId'].apply(lambda x: userid2idx[x])
df['movieId'] = df['movieId'].apply(lambda x: movieid2idx[x])

# apply the lambda function on each user
split = np.random.rand(len(df))<0.8
train = df[split]
valid = df[~split]
print(train.shape,valid.shape)
# We take 80% for training 
n_movies=len(df['movieId'].unique())
n_users=len(df['userId'].unique())
n_latent_factors=64  # hyperparamter to deal with. 
user_input=Input(shape=(1,),name='user_input',dtype='int64')
user_input
user_embedding=Embedding(n_users,n_latent_factors,name='user_embedding')(user_input)
user_vec =Flatten(name='FlattenUsers')(user_embedding)
movie_input=Input(shape=(1,),name='movie_input',dtype='int64')
movie_embedding=Embedding(n_movies,n_latent_factors,name='movie_embedding')(movie_input)
movie_vec=Flatten(name='FlattenMovies')(movie_embedding)
sim=dot([user_vec,movie_vec],name='Simalarity-Dot-Product',axes=1)
model =keras.models.Model([user_input, movie_input],sim)<jupyter_output>WARNING:tensorflow:From C:\ProgramData\Miniconda3\envs\ntds_2019\lib\site-packages\keras\backend\tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.<jupyter_text>Creates a model that we then try to fit on data.<jupyter_code>model.compile(optimizer=Adam(lr=1e-4),loss='mse')
train.shape
batch_size=128
epochs=50
History = model.fit([train.userId,train.movieId],train.rating, batch_size=batch_size,
                              epochs =epochs, validation_data = ([valid.userId,valid.movieId],valid.rating),
                              verbose = 1)<jupyter_output>WARNING:tensorflow:From C:\ProgramData\Miniconda3\envs\ntds_2019\lib\site-packages\keras\backend\tensorflow_backend.py:986: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.

WARNING:tensorflow:From C:\ProgramData\Miniconda3\envs\ntds_2019\lib\site-packages\keras\backend\tensorflow_backend.py:973: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.

Train on 79940 samples, validate on 20060 samples
Epoch 1/50
79940/79940 [==============================] - 2s 20us/step - loss: 13.7379 - val_loss: 13.6820
Epoch 2/50
79940/79940 [==============================] - 1s 16us/step - loss: 13.7267 - val_loss: 13.6717
Epoch 3/50
79940/79940 [==============================] - 1s 16us/step - loss: 13.6900 - val_loss: 13.6107
Epoch 4/50
79940/79940 [==============================] - 1s 18us/step - loss: 13.5398 - val_loss: 13.3514
Epoch 5/50
79940/79940 [==============================] - 1s 17us/step - loss: 13.0607 - val_loss: 12.6330
Epoch 6/5[...]<jupyter_text>Model Performance<jupyter_code>from pylab import rcParams
rcParams['figure.figsize'] = 10, 5
import matplotlib.pyplot as plt
plt.plot(History.history['loss'] , 'g')
plt.plot(History.history['val_loss'] , 'b')
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.grid(True)
plt.show()<jupyter_output><empty_output><jupyter_text>Neural Network<jupyter_code>n_latent_factors=50
n_movies=len(df['movieId'].unique())
n_users=len(df['userId'].unique())
user_input=Input(shape=(1,),name='user_input',dtype='int64')
user_embedding=Embedding(n_users,n_latent_factors,name='user_embedding')(user_input)
user_vec=Flatten(name='FlattenUsers')(user_embedding)
user_vec=Dropout(0.40)(user_vec)
movie_input=Input(shape=(1,),name='movie_input',dtype='int64')
movie_embedding=Embedding(n_movies,n_latent_factors,name='movie_embedding')(movie_input)
movie_vec=Flatten(name='FlattenMovies')(movie_embedding)
movie_vec=Dropout(0.40)(movie_vec)
sim=dot([user_vec,movie_vec],name='Simalarity-Dot-Product',axes=1)<jupyter_output><empty_output><jupyter_text>Model Architecture<jupyter_code>nn_inp=Dense(96,activation='relu')(sim)
nn_inp=Dropout(0.4)(nn_inp)
# nn_inp=BatchNormalization()(nn_inp)
nn_inp=Dense(1,activation='relu')(nn_inp)
nn_model =keras.models.Model([user_input, movie_input],nn_inp)
nn_model.summary()

nn_model.compile(optimizer=Adam(lr=1e-3),loss='mse')
batch_size=128
epochs=20
History = nn_model.fit([train.userId,train.movieId],train.rating, batch_size=batch_size,
                              epochs =epochs, validation_data = ([valid.userId,valid.movieId],valid.rating),
                              verbose = 1)<jupyter_output>Train on 80044 samples, validate on 19956 samples
Epoch 1/20
80044/80044 [==============================] - 4s 54us/step - loss: 3.0275 - val_loss: 1.2605
Epoch 2/20
80044/80044 [==============================] - 3s 40us/step - loss: 1.3392 - val_loss: 1.1010
Epoch 3/20
80044/80044 [==============================] - 3s 35us/step - loss: 1.1021 - val_loss: 0.9949
Epoch 4/20
80044/80044 [==============================] - 3s 36us/step - loss: 0.9811 - val_loss: 0.9500
Epoch 5/20
80044/80044 [==============================] - 3s 33us/step - loss: 0.9201 - val_loss: 0.9314
Epoch 6/20
80044/80044 [==============================] - 3s 32us/step - loss: 0.8689 - val_loss: 0.9218
Epoch 7/20
80044/80044 [==============================] - 3s 34us/step - loss: 0.8428 - val_loss: 0.9129
Epoch 8/20
80044/80044 [==============================] - 3s 36us/step - loss: 0.8109 - val_loss: 0.9074
Epoch 9/20
80044/80044 [==============================] - 3s 36us/step - loss: 0.7911 - val_loss: 0.9026
Epoch[...]<jupyter_text>Test And useful Functions<jupyter_code>model.predict([np.array([163]), np.array([163])])
column_list = ['movie id', 'movie title', 'release date', 'video release date','IMDb URL','unknown','Action','Adventure','Animation','Childrens','Comedy','Crime','Documentary','Drama','fantasy','Film-Noir','Horror','Musical','Mystery','Romance','Sci-Fi','Thriller','War',' Western']
item_content =  pd.read_csv('ml-100k/u.item', delimiter='\|', names=column_list, index_col=column_list[0], engine="python")

def get_movie_name(movie_id, item_content):
        """
        Get the name of a movie from it's id.
        Args:
            movie_id = the id for the movie
        
        Returns:
            The movie title for the specified movie_id.
        """

        movies = item_content.loc[movie_id, 'movie title']
        return movies
    
def predict_for_user(user_id, movie_id,model):    
    """
    Predict if a user will like a specific movie.
    Args:
        user_id = The user's user_id.
        movie_id = The movie's movie_id.
    Returns:
        A prediction of how the user would rate the movie.
    """

    prediction = model.predict([np.array([user_id]), np.array([movie_id])])
    #prediction = round(prediction[0][0])

    # astype(int) converts the numpy float32 value to an integer
    return prediction.astype(float)
predict_for_user(163,163,model)
get_movie_name(163,item_content)
movie_data = np.array(list(set(df.movieId)))
movie_data[:5]
user = np.array([189 for i in range(len(movie_data))])
user[:5]
predictions = model.predict([user, movie_data])
predictions = np.array([a[0] for a in predictions])
recommended_movie_ids = (-predictions).argsort()[:5]
print(recommended_movie_ids)
print(predictions[recommended_movie_ids])
column_list = ['movieId', 'movie title', 'release date', 'video release date','IMDb URL','unknown','Action','Adventure','Animation','Childrens','Comedy','Crime','Documentary','Drama','fantasy','Film-Noir','Horror','Musical','Mystery','Romance','Sci-Fi','Thriller','War',' Western']
movies =  pd.read_csv(r'ml-100k/u.item', delimiter='\|', names=column_list, index_col=column_list[0], engine="python")
movies.head()
recommandation(model,len(df['movieId'].unique()),189)
print(predict_for_user(189,318,model))
get_movie_name(318,item_content)
print(predict_for_user(189,190,model))
get_movie_name(190,item_content)
print(predict_for_user(189,493,model))
get_movie_name(493,item_content)<jupyter_output>[[1.74413502]]
{'fraction_non_alphanumeric': 0.16675485008818342, 'fraction_numerical': 0.050793650793650794, 'mean_word_length': 6.177848101265822, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '636526', 'license': 'MIT', 'n_tokens_mistral': 4351, 'n_tokens_neox': 3864, 'n_words': 769, 'path': 'Model_based_recommandation.ipynb', 'repo_name': 'aurkinet/Group-16', 'size': '11340'}