<jupyter_start><jupyter_text>Complete Script for Analyzing Movie Data <NAME> and <NAME><jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
warnings.filterwarnings('ignore')
results_path = '../Data/result_df.csv'
movies_path = '../Data/movies_df.csv'
results = pd.read_csv(results_path)
movies = pd.read_csv(movies_path)
movies = pd.read_csv(movies_path)
results = pd.read_csv(results_path)
final = pd.merge(movies, results, left_index=True, right_index=True)<jupyter_output><empty_output><jupyter_text>Final DataFrame with Combined Data<jupyter_code>final.head()
pair = sns.pairplot(final.drop(['title'],axis=1), vars = ['person','car', 'pizza','knife','truck'], hue= 'Action')
genre_df = final[['Family', 'Animation',
       'History', 'Documentary', 'Adventure', 'Western', 'Crime', 'Drama',
       'Horror', 'Science Fiction', 'Romance', 'War', 'Mystery', 'Fantasy',
       'Action', 'TV Movie', 'Thriller', 'Comedy', 'Music']]
gct = pd.DataFrame(genre_df.apply(pd.Series.value_counts).loc[1].transpose())
gct = gct.reset_index()
gct.columns
gct.columns = ['genres', 'count']
gct.sort_values(by='count', ascending=False, inplace=True)
plt.figure(figsize=(15,20)) 
sns.barplot(x='count',y ='genres', data=gct)<jupyter_output><empty_output><jupyter_text>YOLO Data Approach<jupyter_code>y = final.drop(final[['id','title']], axis = 1)
y = y[['Mystery', 'Animation', 'Music', 'History', 'Comedy', 'Science Fiction',
       'Family', 'Fantasy', 'Romance', 'Horror', 'War', 'Documentary',
       'TV Movie', 'Adventure', 'Drama', 'Western', 'Thriller', 'Action',
       'Crime']]
X = final.drop(final[['Mystery', 'Animation', 'Music', 'History', 'Comedy', 'Science Fiction',
       'Family', 'Fantasy', 'Romance', 'Horror', 'War', 'Documentary',
       'TV Movie', 'Adventure', 'Drama', 'Western', 'Thriller', 'Action',
       'Crime','overview','release_date']], axis = 1)
X = X.drop(X[['title','id']], axis = 1)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
from sklearn.svm import SVC
from sklearn.multiclass import OneVsRestClassifier
model = OneVsRestClassifier(SVC(gamma='auto'))
model.fit(X_train,y_train)
predicted = model.predict(X_test)
from sklearn.metrics import confusion_matrix, classification_report
print(f'{classification_report(y_test,predicted)}')<jupyter_output>precision    recall  f1-score   support

           0       0.00      0.00      0.00       262
           1       0.00      0.00      0.00       273
           2       0.00      0.00      0.00        95
           3       0.00      0.00      0.00       142
           4       0.35      0.11      0.17      1132
           5       0.22      0.02      0.04       367
           6       0.00      0.00      0.00       353
           7       0.13      0.01      0.01       332
           8       0.16      0.01      0.02       569
           9       0.11      0.01      0.02       393
          10       0.00      0.00      0.00       106
          11       0.00      0.00      0.00        51
          12       0.00      0.00      0.00        31
          13       0.24      0.03      0.05       511
          14       0.46      0.25      0.32      1540
          15       0.00      0.00      0.00        60
          16       0.27      0.05      0.08       900
          17       0.32    [...]<jupyter_text>Word2Vec Approach<jupyter_code>movies_path = '../Data/movies_df.csv'
movies = pd.read_csv(movies_path)
movies.columns
movies = movies.drop(['release_date'], axis=1)
movies = movies.drop(['overview'],axis=1)
from gensim.models import Word2Vec
from gensim.models import KeyedVectors
filename = '../GoogleNews-vectors-negative300.bin'
model = KeyedVectors.load_word2vec_format(filename, binary=True)
stoplist = set('for a of the and to in : &'.split())
punctuation = set('; / " . , :'.split())
def most_likely(title, genres = ['Mystery',
       'Animation', 'Music', 'History', 'Comedy', 'Science', 'Family',
       'Fantasy', 'Romance', 'Horror', 'War', 'Documentary', 'TV',
       'Adventure', 'Drama', 'Western', 'Thriller', 'Action', 'Crime']):
    
    broken_title = title.split(' ')

    print(broken_title)
    dists = []
    d = [0] * len(genres)
    for word in broken_title:
        if word.lower() in stoplist:
            continue
        
        else:
            for stop in punctuation:
                if stop in word:
                    word = word.split(stop)[0]
                
            for genre in genres:
                d[genres.index(genre)] += model.distance(word.lower(), genre)
            
    dists = [x / len(broken_title) for x in d]
    
    
    print(sorted(zip(dists,genres), reverse = False)[:3])


genres = ['Mystery',
       'Animation', 'Music', 'History', 'Comedy', 'Science', 'Family',
       'Fantasy', 'Romance', 'Horror', 'War', 'Documentary', 'TV',
       'Adventure', 'Drama', 'Western', 'Thriller', 'Action', 'Crime']
most_likely('how to train your dragon:', genres)<jupyter_output><empty_output><jupyter_text>TF-IDF Approach<jupyter_code>import re
import nltk
df = pd.read_csv('../Data/movies_df.csv')
def clean(text): # use regular expression to remove specific characters
    text = re.sub("\'", " ", text)
    text = re.sub("[^a-zA-Z]"," ",text)
    text = ' '.join(text.split())
    text = text.lower()
    return text
df['clean_overview'] = df['overview'].astype(str).apply(lambda x: clean(x))
stpwrds = set(nltk.corpus.stopwords.words('english'))
def remove_stops(text):
    cleaned = [w for w in text.split() if not w in stpwrds]
    return ' '.join(cleaned)
df['clean_overview'] = df['clean_overview'].astype(str).apply(lambda x: remove_stops(x))
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=10000)
X = df['clean_overview']
y = df[['Family', 'Animation',
       'History', 'Documentary', 'Adventure', 'Western', 'Crime', 'Drama',
       'Horror', 'Science Fiction', 'Romance', 'War', 'Mystery', 'Fantasy',
       'Action', 'TV Movie', 'Thriller', 'Comedy', 'Music']]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)

tfidf_xtrain = tfidf_vectorizer.fit_transform(X_train)
tfidf_xtest = tfidf_vectorizer.transform(X_test)
from sklearn.svm import SVC
from sklearn.multiclass import OneVsRestClassifier
from sklearn.metrics import classification_report
model = OneVsRestClassifier(SVC(gamma = 'auto',kernel = 'linear'))
model.fit(tfidf_xtrain, y_train)
predictions = model.predict(tfidf_xtest)
print(classification_report(predictions,y_test))<jupyter_output><empty_output><jupyter_text>Use Multilabel Binarizer Instead<jupyter_code>def make_genre_list(s):
    genre_list = ['Family', 'Animation',
       'History', 'Documentary', 'Adventure', 'Western', 'Crime', 'Drama',
       'Horror', 'Science Fiction', 'Romance', 'War', 'Mystery', 'Fantasy',
       'Action', 'TV Movie', 'Thriller', 'Comedy', 'Music']
    gen = []
    for g in genre_list:
        if s.loc[g] == 1:
            gen.append(g)
    return gen
df['genre_list'] = df.apply(lambda row : make_genre_list(row), axis = 1)
from sklearn.preprocessing import MultiLabelBinarizer
multilabel_binarizer = MultiLabelBinarizer()
multilabel_binarizer.fit(df['genre_list'])

X = df['clean_overview']
y = multilabel_binarizer.transform(df['genre_list'])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
tfidf_xtrain = tfidf_vectorizer.fit_transform(X_train)
tfidf_xtest = tfidf_vectorizer.transform(X_test)
model = OneVsRestClassifier(SVC(gamma = 'auto', kernel = 'linear'))
model.fit(tfidf_xtrain, y_train)
predictions = model.predict(tfidf_xtest)
print(classification_report(predictions,y_test))
from sklearn.linear_model import LogisticRegression
logovr = OneVsRestClassifier(LogisticRegression())
logovr.fit(tfidf_xtrain, y_train)
predicted = logovr.predict(tfidf_xtest)
print(classification_report(predicted, y_test))
mbi = multilabel_binarizer.inverse_transform(predicted)
y_test_df = pd.DataFrame()
y_test_df['genres'] = y_test.apply(lambda row : make_genre_list(row), axis = 1)
ytgenres = list(y_test_df['genres'])
for i in range(len(mbi)):
    print(f'{mbi[i]}\t\t\t\t{ytgenres[i]}')<jupyter_output><empty_output><jupyter_text>Dominant color approach<jupyter_code>from matplotlib import image
from scipy.cluster.vq import whiten
from scipy.cluster.vq import kmeans
from timeit import default_timer as timer

def get_dominant_color(id):
    start = timer()
    img = image.imread(r'../Data/Images/' + str(id) + '.jpg')
    np_img = np.reshape(img, (img.shape[0]*img.shape[1], img.shape[2]))
    np_flat_img = whiten(np_img)
    cluster_centers, distortion = kmeans(np_flat_img, 2)
    
    red_std = np_img[:, 0].std()
    green_std = np_img[:, 1].std()
    blue_std =  np_img[:, 0].std()
    
    results = {}
    for index, cluster_center in enumerate(cluster_centers):
        int_color = int('{:02x}{:02x}{:02x}'.format(
            int(cluster_center[0] * red_std), 
            int(cluster_center[1] * green_std), 
            int(cluster_center[2] * blue_std)), 16)
        key = 'color_{}'.format(index)
        results[key] = int_color
    end = timer()
    print('{0}: {1:.2f}s'.format(id, end - start))
    return results
    
movies = pd.read_csv(r'../Data/movies_df.csv')
temp = movies[:25].apply(lambda x: get_dominant_color(x['id']), axis=1, result_type='expand')
df = pd.merge(movies, temp, left_index=True, right_index=True, )
df.head()
sns.pairplot(df, vars=['color_0', 'color_1'], hue='Drama')
from sklearn.model_selection import train_test_split
df.columns
df.describe()
y = df[['Action']]
X = df[['color_0', 'color_1']]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, stratify=y)
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import SVC
model = OneVsRestClassifier(SVC(gamma='auto'))
model.fit(X_train,y_train)
predictions = model.predict(X_test)
from sklearn.metrics import confusion_matrix, classification_report
print(classification_report(y_test,predictions))<jupyter_output>precision    recall  f1-score   support

           0       0.62      1.00      0.77         5
           1       0.00      0.00      0.00         3

   micro avg       0.62      0.62      0.62         8
   macro avg       0.31      0.50      0.38         8
weighted avg       0.39      0.62      0.48         8
{'fraction_non_alphanumeric': 0.1461910634664643, 'fraction_numerical': 0.03415235746134143, 'mean_word_length': 3.3454245671887883, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '191793', 'license': 'MIT', 'n_tokens_mistral': 3946, 'n_tokens_neox': 3724, 'n_words': 781, 'path': 'Final/.ipynb_checkpoints/Complete_Script_Aaron_Christoph-checkpoint.ipynb', 'repo_name': 'AaronRiegel/GenrePrediction', 'size': '10560'}