<jupyter_start><jupyter_text>Prepare Datasets<jupyter_code>import pandas as pd
import numpy as np
#loading the movie dataset
movies = pd.read_csv('/content/tmdb_5000_movies.csv')
#loading the credit dataset
credits = pd.read_csv('/content/tmdb_5000_credits.csv')
credits.columns = ['id', 'title', 'cast', 'crew']
#combine the 2 datasets together 
movies = movies.merge(credits, on = 'id')<jupyter_output><empty_output><jupyter_text>Cleaning Data and Add Features<jupyter_code>import warnings
warnings.filterwarnings('ignore')

# inlude the necessary columns from the dataset. 
data = movies[['id', 'original_title', 'genres', 'keywords', 'overview',  'cast', 'crew']]

# define the features in the datasets
from ast import literal_eval
features = ['keywords','genres', 'cast', 'crew']
for i in features:
    data[i] = movies[i].apply(literal_eval)
    
 # Extract list of genres
def list_genres(x):
    list = [d['name'] for d in x]
    return(list)
data['genres'] = data['genres'].apply(list_genres)

# Extract top 3 cast members
def list_cast(x):
    list = [d['name'] for d in x]
    if len(list) > 3:
        list = list[:3]
    return(list)
data['cast'] = data['cast'].apply(list_cast)

# Extract top 5 keywords
def list_keywords(x):
    list = [d['name'] for d in x]
    if len(list) > 5:
        list = list[:5]
    return(list)
data['keywords'] = data['keywords'].apply(list_keywords)

# Extract director
def get_director(x):
    for i in x:
        if i['job'] == 'Director':
            return i['name']
    return np.nan
data['director'] = data['crew'].apply(get_director)

# Drop the column crew 
data = data.drop('crew', axis = 1)

# Clean features of the dataset
def clean_feat(x):
    if isinstance(x, list):
        return [i.lower().replace(" ","") for i in x]
    else:
        if isinstance(x, str):
            return x.lower().replace(" ", "")
        else:
            return ''
          

 #redefine the features in the data 
features = ['keywords', 'genres', 'cast', 'director']
for i in features:
    data[i] = data[i].apply(clean_feat)
    

data.head()
missing = data.columns[data.isnull().any()]
data[missing].isnull().sum().to_frame()
# Replace NaN from overview with an empty string
data['overview'] = data['overview'].fillna('')

! pip install rake_nltk
from rake_nltk import Rake
# Initialize empty column
data['createtags'] = ''

# function to get keywords from a text
def get_keywords(x):
    plot = x
    
    # initialize Rake using english stopwords from NLTK, and all punctuation characters
    rake = Rake()
    
    # extract keywords from text
    rake.extract_keywords_from_text(plot)
    
    # get dictionary with keywords and scores
    scores = rake.get_word_degrees()
    
    # return new keywords as list, ignoring scores
    return(list(scores.keys()))

# Apply function to generate keywords
data['createtags'] = data['overview'].apply(get_keywords)

data_keys = pd.DataFrame() 

data_keys['title'] = data['original_title']
data_keys['keywords'] = ''

def  tags_package(x):
    return(' '.join(x['genres']) + ' ,' + ' '.join(x['keywords']) + ' ,' +  ' '.join(x['cast']) + 
           ' ,' + ' '.join(x['director']) + ' ,' + ' '.join(x['createtags']))
data_keys['keywords'] = data.apply(tags_package, axis = 1)

data_keys.head()
from sklearn.feature_extraction.text import CountVectorizer

# create count matrix
cv = CountVectorizer()
cv_mx = cv.fit_transform(data_keys['keywords'])


from sklearn.metrics.pairwise import cosine_similarity
# create cosine similarity matrix
cosine_sim = cosine_similarity(cv_mx, cv_mx)

print(cosine_sim)
# create list of indices for later matching
indices = pd.Series(data_keys.index, index = data_keys['title'])<jupyter_output><empty_output><jupyter_text>Result Testing(Recommendation)<jupyter_code>def movies_recommendation(title, n = 10, cosine_sim = cosine_sim):
    movies = []
    
    # retrieve matching movie title index
    if title not in indices.index:
        print("No such movie found")
        return
    else:
        idx = indices[title]
    
    # cosine similarity scores of movies in descending order
    scores = pd.Series(cosine_sim[idx]).sort_values(ascending = False)
    
    # top n most similar movies indexes
    # use 1:n because 0 is the same movie entered
    top_n_idx = list(scores.iloc[1:n].index)
        
    return data_keys['title'].iloc[top_n_idx]
print(movies_recommendation('The Avengers' ,n=12))<jupyter_output>7                   Avengers: Age of Ultron
174                     The Incredible Hulk
511                                   X-Men
85      Captain America: The Winter Soldier
79                               Iron Man 2
26               Captain America: Civil War
169      Captain America: The First Avenger
31                               Iron Man 3
68                                 Iron Man
4401                    The Helix... Loaded
182                                 Ant-Man
Name: title, dtype: object
{'fraction_non_alphanumeric': 0.12489894907033144, 'fraction_numerical': 0.010711398544866612, 'mean_word_length': 3.1798986486486487, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '20651', 'license': 'BSD-4-Clause-UC', 'n_tokens_mistral': 1629, 'n_tokens_neox': 1537, 'n_words': 433, 'path': 'ass.2/python/CB_recommender.ipynb', 'repo_name': 'ShuyanCao/Ass2_Jianyu.Zhang_Shuyan.Cao', 'size': '4948'}