<jupyter_start><jupyter_text>Content Based Recommender System<jupyter_code>import numpy as np 
import pandas as pd
movies = pd.read_csv('data/tmdb_5000_movies.csv')
credits = pd.read_csv('data/tmdb_5000_credits.csv')
movies.head(2)
movies.shape
credits.head()
credits.shape
movies = movies.merge(credits,on='title')
movies.head(2)
movies.shape
# Keeping important columns for recommendation
movies = movies[['movie_id','title','overview','genres','keywords','cast','crew']]
movies.head(2)
movies.shape
movies.isnull().sum()
movies.dropna(inplace=True)
movies.isnull().sum()
movies.shape
movies.duplicated().sum()
# handle genres

movies.iloc[0]['genres']
import ast #for converting str to list

def convert(text):
    L = []
    for i in ast.literal_eval(text):
        L.append(i['name']) 
    return L
movies['genres'] = movies['genres'].apply(convert)
movies.head()
# handle keywords
movies.iloc[0]['keywords']
movies['keywords'] = movies['keywords'].apply(convert)
movies.head()
# handle cast
movies.iloc[0]['cast']
# Here i am just keeping top 3 cast

def convert_cast(text):
    L = []
    counter = 0
    for i in ast.literal_eval(text):
        if counter < 3:
            L.append(i['name'])
        counter+=1
    return L
movies['cast'] = movies['cast'].apply(convert_cast)
movies.head()
# handle crew

movies.iloc[0]['crew']
def fetch_director(text):
    L = []
    for i in ast.literal_eval(text):
        if i['job'] == 'Director':
            L.append(i['name'])
            break
    return L
movies['crew'] = movies['crew'].apply(fetch_director)
movies.head()
# handle overview (converting to list)

movies.iloc[0]['overview']
movies['overview'] = movies['overview'].apply(lambda x:x.split())
movies.sample(4)
movies.iloc[0]['overview']
# now removing space like that 
'<NAME>'
'AnnaKendrick'

def remove_space(L):
    L1 = []
    for i in L:
        L1.append(i.replace(" ",""))
    return L1

movies['cast'] = movies['cast'].apply(remove_space)
movies['crew'] = movies['crew'].apply(remove_space)
movies['genres'] = movies['genres'].apply(remove_space)
movies['keywords'] = movies['keywords'].apply(remove_space)
movies.head()
# Concatinate all
movies['tags'] = movies['overview'] + movies['genres'] + movies['keywords'] + movies['cast'] + movies['crew']
movies.head()
movies.iloc[0]['tags']
# droping those extra columns
new_df = movies[['movie_id','title','tags']]
new_df.head()
# Converting list to str
new_df['tags'] = new_df['tags'].apply(lambda x: " ".join(x))
new_df.head()
new_df.iloc[0]['tags']
# Converting to lower case
new_df['tags'] = new_df['tags'].apply(lambda x:x.lower())
new_df.head()
new_df.iloc[0]['tags']
import nltk
from nltk.stem import PorterStemmer
ps = PorterStemmer()
def stems(text):
    T = []
    
    for i in text.split():
        T.append(ps.stem(i))
    
    return " ".join(T)
new_df['tags'] = new_df['tags'].apply(stems)
new_df.iloc[0]['tags']
from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer(max_features=5000,stop_words='english')

vector = cv.fit_transform(new_df['tags']).toarray()
vector[0]
vector.shape
len(cv.get_feature_names())
from sklearn.metrics.pairwise import cosine_similarity
similarity = cosine_similarity(vector)
similarity.shape
# similarity
new_df[new_df['title'] == 'The Lego Movie'].index[0]
def recommend(movie):
    index = new_df[new_df['title'] == movie].index[0]
    distances = sorted(list(enumerate(similarity[index])),reverse=True,key = lambda x: x[1])
    for i in distances[1:6]:
        print(new_df.iloc[i[0]].title)
recommend('Spider-Man 2')
import pickle
pickle.dump(new_df,open('artifacts/movie_list.pkl','wb'))
pickle.dump(similarity,open('artifacts/similarity.pkl','wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.174403005097934, 'fraction_numerical': 0.011000804936946607, 'mean_word_length': 5.276094276094276, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '240220', 'license': 'MIT', 'n_tokens_mistral': 1387, 'n_tokens_neox': 1330, 'n_words': 254, 'path': 'Movie Recommender System Data Analysis.ipynb', 'repo_name': 'THIRU-AIML/system-recom', 'size': '3734'}