<jupyter_start><jupyter_text>Movie Description Based Recommender<jupyter_code># filling missing values in tagline and description columns
smd['tagline'] = smd['tagline'].fillna('')  # fill the empty data points in tagline column
smd['description'] = smd['overview'] + smd['tagline']  # Combining columns overview and tagline
smd['description'] = smd['description'].fillna('')
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.metrics.pairwise import linear_kernel, cosine_similarity
from surprise import Reader, Dataset, SVD
from surprise.model_selection import cross_validate
# Converts a collection of raw documents to a matrix of TF-IDF features
tf = TfidfVectorizer(analyzer='word', ngram_range=(1, 2), min_df = 0, stop_words='english')
tfidf_matrix = tf.fit_transform(smd['description'])
tfidf_matrix.shape
cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)
cosine_sim[0]
# We now have a pairwise cosine similarity matrix for all the movies in our dataset. 
# The next step is to write a function that returns the 30 most similar movies based 
# on the cosine similarity score.

smd = smd.reset_index()
titles = smd['title']  # Defining a new variable title
indices = pd.Series(smd.index, index = smd['title'])  # Defining a new dataframe indices
smd.head()
# Defining a function that returns 30 most similar movied bases on the cosine 
# similarity score
def get_recommendations(title):
    idx = indices[title]  # Defining a variable with indices
    sim_scores = list(enumerate(cosine_sim[idx]))
    sim_scores = sorted(sim_scores, key = lambda x: x[1], reverse = True)
    sim_scores = sim_scores[1: 31]  # Taking the 30 most similar movies
    movie_indices = [i[0] for i in sim_scores]
    return titles.iloc[movie_indices]  # returns the title based on movie indices
get_recommendations('Transformers').head(10)
# Our system is able to identify it as a Batman film and subsequently recommend 
# other Batman films as its top recommendations.
get_recommendations('The Dark Knight').head(10)<jupyter_output><empty_output><jupyter_text>Metadata Based RecommenderWe will build a more sophisticated recommender that takes genre, keywords, cast and crew into consideration.To build our standard metadata based content recommender, we will need to merge our current dataset with the crew and the keyword datasets.<jupyter_code>credits = pd.read_csv('/home/jay/Desktop/Movie Recommender System/credits.csv')
keywords = pd.read_csv('/home/jay/Desktop/Movie Recommender System/keywords.csv')
# Converting the keywords's id column to integer
keywords['id'] = keywords['id'].astype('int')
credits['id'] = credits['id'].astype('int')
md['id'] = md['id'].astype('int')
md.shape
# Merging teh credits and keywords DataFrame with md DataFrame
md = md.merge(credits, on = 'id')
md = md.merge(keywords, on = 'id')
# Creating a small movies DataFrame from the links_small
smd = md[md['id'].isin(links_small)]
smd.shape
# We now have our cast, crew, genres and credits, all in one dataframe.
from ast import literal_eval

smd['cast'] = smd['cast'].apply(literal_eval)
smd['crew'] = smd['crew'].apply(literal_eval)
smd['keywords'] = smd['keywords'].apply(literal_eval)
smd['cast_size'] = smd['cast'].apply(lambda x: len(x))  # Storing the cast_size
smd['crew_size'] = smd['crew'].apply(lambda x: len(x))  # Storing the crew_size
# Defining a function that gets the director's name 
def get_director(x):
    for i in x:
        if i['job'] == 'Director':
            return i['name']
    return np.nan
# Applying the get_director function to the crew column to create smd['director'] column
smd['director'] = smd['crew'].apply(get_director)
# Arbitrarily we will choose the top 3 actors that appear in the credits list. 
smd['cast'] = smd['cast'].apply(lambda x: [i['name'] for i in x] if isinstance(x, list) else [])
smd['cast'] = smd['cast'].apply(lambda x: x[:3] if len(x) >= 3 else x)  # Taking the top 3 actor from cast
smd['keywords'] = smd['keywords'].apply(lambda x: [i['name'] for i in x] if isinstance(x, list) else [])
# Strip Spaces and Convert to Lowercase from all our features. 
smd['cast'] = smd['cast'].apply(lambda x: [str.lower(i.replace(" ", "")) for i in x])
# Mention Director 3 times to give it more weight relative to the entire cast.
smd['director'] = smd['director'].astype('str').apply(lambda x: str.lower(x.replace(" ", "")))
smd['director'] = smd['director'].apply(lambda x: [x, x, x])
# Calculating the frequent counts of every keyword that appears in the dataset
s = smd.apply(lambda x: pd.Series(x['keywords']), axis = 1).stack().reset_index(level = 1, drop = True)
s.name = 'keywords'
s = s.value_counts()
s[: 5]
# Removing keywords that occur only once
s = s[s > 1]
# We will convert every word to its stem so that words such as Dogs and Dog 
# are considered the same.
from nltk.stem.snowball import SnowballStemmer
stemmer = SnowballStemmer('english')
stemmer.stem('dogs')
# Defining function filter_keywords() to create a list of words
def filter_keywords(x):
    words = []
    for i in x:
        if i in s:
            words.append(i)
    return words
# Converting the entries in keywords into stemmed words
smd['keywords'] = smd['keywords'].apply(filter_keywords)  #Applying the filter_keywords() function to keywords column
smd['keywords'] = smd['keywords'].apply(lambda x: [stemmer.stem(i) for i in x])  # Converting the entries in keywords column into stemmed words
smd['keywords'] = smd['keywords'].apply(lambda x: [str.lower(i.replace(" ", "")) for i in x])  # Removing spaces and converting the entries into lower case
smd.head(5)
# Combining 'keywords', 'cast', 'director' and genres columns in 'soup'
smd['soup'] = smd['keywords'] + smd['cast'] + smd['director'] + smd['genres']
smd['soup'] = smd['soup'].apply(lambda x: ' '.join(x))
# Convert a collection of text documents to a matrix of token counts
count = CountVectorizer(analyzer = 'word', ngram_range = (1, 2), min_df = 0, stop_words = 'english')
count_matrix = count.fit_transform(smd['soup'])
from sklearn.metrics.pairwise import linear_kernel, cosine_similarity
cosine_sim = cosine_similarity(count_matrix, count_matrix)
smd = smd.reset_index()  # Reseting the indices within smd
titles = smd['title']  # Storing smd['title'] in titles variable
indices = pd.Series(smd.index, index = smd['title'])    # Creating a DataFrame of indices
get_recommendations('The Matrix').head(10)
get_recommendations('Divergent').head(10)
def improved_recommendations(title):
    idx = indices[title]
    sim_scores = list(enumerate(cosine_sim[idx]))
    sim_scores = sorted(sim_scores, key = lambda x: x[1], reverse = True)
    sim_score = sim_scores[1:26]
    movie_indices = [i[0] for i in sim_scores]
    
    movies = smd.iloc[movie_indices][['title', 'vote_count', 'vote_average', 'year']]
    vote_counts = movies[movies['vote_count'].notnull()]['vote_count'].astype('int')
    vote_averages = movies[movies['vote_average'].notnull()]['vote_average'].astype('int')
    C = vote_averages.mean()
    m = vote_counts.quantile(0.60)
    qualified = movies[(movies['vote_count'] >= m) & (movies['vote_count'].notnull()) & (movies['vote_average'].notnull())]
    qualified = movies[(movies['vote_count'] >= m) & (movies['vote_count'].notnull()) & (movies['vote_average'].notnull())]
    qualified['vote_count'] = qualified['vote_count'].astype('int')
    qualified['vote_average'] = qualified['vote_average'].astype('int')
    qualified['wr'] = qualified.apply(weighted_rating, axis=1)
    qualified = qualified.sort_values('wr', ascending=False).head(10)
    return qualified
vote_counts = md[md['vote_count'].notnull()]['vote_count'].astype('int')
vote_averages = md[md['vote_average'].notnull()]['vote_average'].astype('int')
C = vote_averages.mean()
C
m = vote_counts.quantile(0.99)
m
def weighted_rating(x):
    v = x['vote_count']
    R = x['vote_average']
    return (v/(v+m) * R) + (m/(m+v) * C)
improved_recommendations('The Shawshank Redemption')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13756085382598926, 'fraction_numerical': 0.006241418050181001, 'mean_word_length': 5.466505246166263, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '333021', 'license': 'MIT', 'n_tokens_mistral': 2680, 'n_tokens_neox': 2543, 'n_words': 767, 'path': 'Movie Recommender System.ipynb', 'repo_name': 'Jaybagrecha/Movie_Recommender_System', 'size': '8011'}