app.py

#!/usr/bin/env python
# coding: utf-8

# In[2]:


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import gradio as gr
#from scipy import stats
from ast import literal_eval
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.metrics.pairwise import linear_kernel, cosine_similarity
from nltk.stem.snowball import SnowballStemmer
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import wordnet
from surprise import Reader, Dataset, SVD
import warnings; warnings.simplefilter('ignore')
import surprise


# In[3]:


path = 'C:/HW/Spring 2022/Deep learning/Project/all csvs'


# In[4]:


md = pd.read_csv(path+'/movies_metadata.csv')
md.head(2)


# <b> Simple rec system <b>
# 

# In[5]:


md['genres'] = md['genres'].fillna('[]').apply(literal_eval).apply(lambda x: [i['name'] for i in x] if isinstance(x, list) else [])
# fillna replaces NaN values with '[]'
# Get just the genres


# Weighted Rating (WR) =  (v/(v+m)*R)+(m/(v+m).C)
# 
# where,
# 
# [1] v is the number of votes for the movie <br>
# [2] m is the minimum votes required to be listed in the chart <br>
# [3] R is the average rating of the movie <br>
# [4] C is the mean vote across the whole report <br>

# In[6]:


vote_counts = md[md['vote_count'].notnull()]['vote_count'].astype(int)
vote_average = md[md['vote_average'].notnull()]['vote_average'].astype(int)

C = np.mean(vote_average)
m = vote_counts.quantile(0.95)

print('The average rating for these movies is: ',C)
print('The minimum votes required to be listed in the chart: ',m)


# In[7]:


# Keeping the year from the date
md['year'] = pd.to_datetime(md['release_date'], errors='coerce').apply(lambda x: str(x).split('-')[0] if x != np.nan else np.nan)


# In[8]:


md['popularity']


# In[9]:


qualified = md[(md['vote_count'] >= m) & (md['vote_count'].notnull()) & (md['vote_average'].notnull())][['title', 'year', 'vote_count', 'vote_average', 'popularity', 'genres']]


# In[10]:


qualified['vote_count'] = qualified['vote_count'].astype(int)
qualified['vote_average'] = qualified['vote_average'].astype(int)
qualified.shape


# In[11]:


def weighted_rating(x):
    v = x['vote_count']
    R = x['vote_average']
    return (v/(v+m) * R) + (m/(m+v) * C)


# In[12]:


qualified['wr'] = qualified.apply(weighted_rating, axis=1)
qualified = qualified.sort_values('wr',ascending = False).head(250)


# In[13]:


s = md.apply(lambda x: pd.Series(x['genres']),axis=1).stack().reset_index(level=1, drop=True)
s.name = 'genre'
gen_md = md.drop('genres', axis=1).join(s)


# In[14]:


def build_chart(genre, percentile=0.85):
    
    df = gen_md[gen_md['genre'] == genre] # Getting gen_md for specific genres
    vote_counts = df[df['vote_count'].notnull()]['vote_count'].astype('int')
    vote_averages = df[df['vote_average'].notnull()]['vote_average'].astype('int')
    C = vote_averages.mean()
    m = vote_counts.quantile(percentile)
    
    
    qualified = df[(df['vote_count'] >= m) & (df['vote_count'].notnull()) & (df['vote_average'].notnull())][['title', 'year', 'vote_count', 'vote_average', 'popularity']]
    qualified['vote_count'] = qualified['vote_count'].astype('int')
    qualified['vote_average'] = qualified['vote_average'].astype('int')
    
    qualified['wr'] = qualified.apply(lambda x: (x['vote_count']/(x['vote_count']+m) * x['vote_average']) + (m/(m+x['vote_count']) * C), axis=1)
    qualified = qualified.sort_values('wr', ascending=False).head(250)
    
    return qualified


# In[15]:


build_chart('Romance')


# <b> Content Based Recommender/ Filtering <b>
# 
# In this section we personalize the movie recommendations, Content Based Recommenders based on:
# 
# Movie Overviews and Taglines <br>
# Movie Cast, Crew, Keywords and Genre
#     

# In[16]:


links = pd.read_csv(path+'/links_small.csv')
links = links[links['tmdbId'].notnull()]['tmdbId'].astype(int)


# In[17]:


md = md.drop([19730, 29503, 35587])


# In[18]:


md['id'] = md['id'].astype('int')


# In[19]:


# Getting the movies that their IDs exist in "links"
smd = md[md['id'].isin(links)]
smd.shape


# In[20]:


smd['tagline'] = smd['tagline'].fillna('')
smd['description'] = smd['overview'] + smd['tagline']
smd['description'] = smd['description'].fillna('')


# <b><font size="3"> This is where things gets exciting!!!!!!!!!<font> <b>
# 
# [1] Convert a collection of raw documents to a matrix of TF-IDF features --  TF-IDF: term frequency–inverse document frequency <br>
# <b>how many times a word appears in a document, and the inverse document frequency of the word across a set of documents?<b> <br>
# 
# [2] ngram_range: All values of n such that min_n <= n <= max_n will be used. For example an ngram_range of (1, 1) means only unigrams, (1, 2) means unigrams and bigrams,  So we're using both unigrams and bigrams <br>
# 
# [3] A 1-gram (or unigram) is a one-word sequence. ... A 2-gram (or bigram) is a two-word sequence of words, like “I love”, “love reading”, or “Analytics Vidhya”. And a 3-gram (or trigram) is a three-word sequence of words like “I love reading”

# In[21]:


tf = TfidfVectorizer(analyzer='word',ngram_range=(1, 2),min_df=0, stop_words='english')
tfidf_matrix = tf.fit_transform(smd['description'])


# In[22]:


cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)


# In[23]:


smd = smd.reset_index()


# In[24]:


titles = smd['title']
indices = pd.Series(smd.index, index=smd['title'])


# In[25]:


tfidf_matrix.shape


# In[34]:


def get_recommendations(title):
    if indices[title].shape ==():
        idx = indices[title]
    else:
        idx = indices[title][0]
    sim = cosine_sim    
    sim_scores = list(enumerate(sim[idx]))
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:5]
    title_idx= [l[0] for l in sim_scores]
    title_rec = [titles[i] for i in title_idx]
    return title_rec


# In[28]:


def greet(name):
    return "Hello " + name + "!!"


# In[27]:


get_recommendations('The Dark Knight',cosine_sim)


# <b> <font size="3"> Adding the metadata to the rec system <font> <b>

# In[42]:


credits = pd.read_csv(path+'/credits.csv')
keywords = pd.read_csv(path+'/keywords.csv')


# In[43]:


keywords['id'] = keywords['id'].astype('int')
credits['id'] = credits['id'].astype('int')
md['id'] = md['id'].astype('int')


# In[44]:


md = md.merge(credits, on = 'id')
md = md.merge(keywords, on = 'id')
smd = md[md['id'].isin(links)]


# In[45]:


smd.shape


# In[46]:


smd['cast'] = smd['cast'].apply(literal_eval)
smd['crew'] = smd['crew'].apply(literal_eval)
smd['keywords'] = smd['keywords'].apply(literal_eval)
smd['cast_size'] = smd['cast'].apply(lambda x: len(x))
smd['crew_size'] = smd['crew'].apply(lambda x: len(x))


# In[47]:


def get_director(x):
    for i in x:
        if i['job'] == 'Director':
            return i['name']
    return np.nan


# In[48]:


smd['director'] = smd['crew'].apply(get_director)
smd['cast'] = smd['cast'].apply(lambda x: [i['name'] for i in x] if isinstance(x,list) else [])
smd['cast'] = smd['cast'].apply(lambda x: x[:3] if len(x)>=3 else x)


# In[49]:


smd['keywords'] = smd['keywords'].apply(lambda x: [i['name'] for i in x] if isinstance(x,list) else [])


# In[50]:


smd['cast'] = smd['cast'].apply(lambda x: [str.lower(i.replace(" ","")) for i in x])


# In[51]:


smd['director'] = smd['director'].astype('str').apply(lambda x: str.lower(x.replace(" ", "")))
smd['director'] = smd['director'].apply(lambda x: [x,x, x])
# we mentioned director 3 times to give it more weight


# In[52]:


s = smd.apply(lambda x: pd.Series(x['keywords']),axis=1).stack().reset_index(level=1, drop=True)
s.name = 'keyword'
s=s.value_counts()
s = s[s>1]


# In[53]:


stemmer = SnowballStemmer('english')


# In[54]:


stemmer.stem('')


# In[55]:


smd['keywords'] = smd['keywords'].apply(lambda x: [i for i in x if i in s])
smd['keywords'] = smd['keywords'].apply(lambda x: [stemmer.stem(i) for i in x])
smd['keywords'] = smd['keywords'].apply(lambda x: [str.lower(i.replace(" ","")) for i in x])


# In[56]:


smd['soup'] = smd['keywords'] + smd['cast'] + smd['director'] + smd['genres']
smd['soup'] = smd['soup'].apply(lambda x: ' '.join(x))


# In[57]:


count = CountVectorizer(analyzer = 'word', ngram_range = (1,2), min_df = 0, stop_words = 'english')
count_matrix = count.fit_transform(smd['soup'])


# In[58]:


cosine_sim2 = linear_kernel(count_matrix, count_matrix)


# In[59]:


smd = smd.reset_index()
titles = smd['title']
indices = pd.Series(smd.index, index=smd['title'])


# In[45]:


cosine_sim2.shape


# In[60]:


def get_recommendations(title,sim):
    if indices[title].shape ==():
        idx = indices[title]
    else:
        idx = indices[title][0]
    sim_scores = list(enumerate(sim[idx]))
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:31]
    title_idx= [l[0] for l in sim_scores]
    title_rec = [titles[i] for i in title_idx]
    return title_rec


# In[62]:


get_recommendations('The Avengers',cosine_sim2)


# <font size="3"> This recommendation system works a lot better than the first, but it doesn't take popularity into account. <font>

# In[75]:


def improved_recommendations(title):
    idx = indices[title]
    sim_scores = list(enumerate(cosine_sim2[idx]))
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:26]
    movie_indices = [i[0] for i in sim_scores]
    
    movies = smd.iloc[movie_indices][['title', 'vote_count', 'vote_average', 'year']]
    vote_counts = movies[movies['vote_count'].notnull()]['vote_count'].astype('int')
    vote_averages = movies[movies['vote_average'].notnull()]['vote_average'].astype('int')
    C = vote_averages.mean()
    m = vote_counts.quantile(0.60)
    qualified = movies[(movies['vote_count'] >= m) & (movies['vote_count'].notnull()) & (movies['vote_average'].notnull())]
    qualified['vote_count'] = qualified['vote_count'].astype('int')
    qualified['vote_average'] = qualified['vote_average'].astype('int')
    qualified['wr'] = qualified.apply(weighted_rating, axis=1)
    qualified = qualified.sort_values('wr', ascending=False).head(10)
    return list(qualified['title'])


# In[76]:


list(improved_recommendations('Mean Girls'))


# In[81]:


iface = gr.Interface(fn=improved_recommendations, title= "Enter movie title for recommendations",inputs="text", outputs=["text",'text','text','text',"text",'text','text','text'], examples = ['The Dark Knight', 'Mean Girls', 'Avatar','The Godfather', 'Top Gun', 'Toy Story'])
iface.launch(share=True)


# In[83]: