<jupyter_start><jupyter_text>How much money can a director lose before they disappear in the industry?  How many box office bombs can an established director endure?<jupyter_code># 20200726 JLCY
# data from https://www.the-numbers.com/

import os
import pandas as pd
import numpy as np
import math
import seaborn as sns
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report

xl = pd.ExcelFile('directors_data.xlsx')
print('number of directors:', len(xl.sheet_names))

eliminate_yr_threshold = 2016

pd.options.mode.chained_assignment = None
if os.path.exists('single_df.csv'): 
    
    combine_df = pd.read_csv('single_df.csv')
    
else: 
    
    append_data = []

    for sheet_name in xl.sheet_names: 

        #print(sheet_name)

        xl_file = pd.read_excel('directors_data.xlsx', sheet_name=sheet_name)

        # combine 1st 2 rows as column names
        combine_col = []

        for ind, col in enumerate(xl_file.columns): 

            if isinstance(xl_file.iloc[0, ind], str): 
                combine_col.append(col + ' ' + xl_file.iloc[0, ind])
            else: 
                combine_col.append(col)

        combine_col = ['Opening Theaters' if i == 'Opening.1 Theaters' else i for i in combine_col]

        xl_file.columns = combine_col

        df = xl_file.iloc[1:-3] # drop 1st row and last 3 summary rows
        df.reset_index(inplace=True)
        df = df.iloc[:, 1:] # drop 1st duplicating column

        # populate data for multiple roles in the same movie
        nan_col = [col for col in df.columns if col not in ['Role']]

        for i in df.index: 

            if type(df['Title'][i]) is float: 

                if math.isnan(df['Title'][i]) and isinstance(df['Role'][i], str): 

                    for col in nan_col: 
                        df[col][i] = df[col][i-1] # set data to the row above

        df['Director'] = sheet_name
        
        df['Release Date'] = pd.to_datetime(df['Release Date'])

        if df['Release Date'].dt.year.unique().max() < eliminate_yr_threshold: 
            df['Stopped Making Movies'] = True
        else: 
            df['Stopped Making Movies'] = False
            
        df['Personal Role Count'] = (np.arange(len(df))+1)[::-1]
        
        df['Personal Director Count'] = np.NaN
        df.loc[df['Role'] == 'Director', 'Personal Director Count'] \
            = (np.arange(len(df.loc[df['Role'] == 'Director']))+1)[::-1]
        
        df['Personal Movie count'] = np.NaN
        df['Personal Movie count'].iloc[-1] = 1
        for i in reversed(df.index[:-1]): 
            if df['Title'][i] != df['Title'][i+1]: 
                df['Personal Movie count'][i] = df['Personal Movie count'][i+1] + 1
            else: 
                df['Personal Movie count'][i] = df['Personal Movie count'][i+1]
        
        df['Profit'] = df['Worldwide Box Office'] - df['Production Budget']
        df.loc[:, 'Profit'] = df['Profit'].astype(float)

        df['Cumulated Profit'] = np.NaN
        df['Cumulated Profit'].iloc[-1] = df['Profit'].iloc[-1]
        for i in reversed(df.index[:-1]): 
            if df['Title'][i] != df['Title'][i+1]: 
                df['Cumulated Profit'][i] = np.nansum([df['Cumulated Profit'][i+1], df['Profit'][i]])
            else: 
                df['Cumulated Profit'][i] = df['Cumulated Profit'][i+1]
                
        df['Profitable Movie Count'] = 0
        df['Unprofitable Movie Count'] = 0
        df['Unknown profit Movie Count'] = 0
        
        if math.isnan(df['Profit'].iloc[-1]): # bottom row
            df['Unknown profit Movie Count'].iloc[-1] += 1
        else: 
            if df['Profit'].iloc[-1] > 0: 
                df['Profitable Movie Count'].iloc[-1] += 1
            else: 
                df['Unprofitable Movie Count'].iloc[-1] += 1
                
        for i in reversed(df.index[:-1]): 
            if df['Title'][i] != df['Title'][i+1]: 
                if math.isnan(df['Profit'][i]): 
                    df['Unknown profit Movie Count'][i] = df['Unknown profit Movie Count'][i+1:].max() + 1
                else: 
                    if df['Profit'][i] > 0: 
                        df['Profitable Movie Count'][i] = df['Profitable Movie Count'][i+1:].max() + 1
                    else: 
                        df['Unprofitable Movie Count'][i] = df['Unprofitable Movie Count'][i+1:].max() + 1
            else: 
                df['Profitable Movie Count'][i] = df['Profitable Movie Count'][i+1]
                df['Unprofitable Movie Count'][i] = df['Unprofitable Movie Count'][i+1]
                df['Unknown profit Movie Count'][i] = df['Unknown profit Movie Count'][i+1]
        
        df['Profitable Movie Director Count'] = 0
        df['Unprofitable Movie Director Count'] = 0
        df['Unknown profit Movie Director Count'] = 0
        
        df.loc[(df['Role'] == 'Director') & (df['Profit'] > 0), 'Profitable Movie Director Count'] \
            = (np.arange(len(df.loc[(df['Role'] == 'Director') & (df['Profit'] > 0)]))+1)[::-1]
        df.loc[(df['Role'] == 'Director') & (df['Profit'] < 0), 'Unprofitable Movie Director Count'] \
            = (np.arange(len(df.loc[(df['Role'] == 'Director') & (df['Profit'] < 0)]))+1)[::-1]
        df.loc[(df['Role'] == 'Director') & (df['Profit'].isnull()), 'Unknown profit Movie Director Count'] \
            = (np.arange(len(df.loc[(df['Role'] == 'Director') & (df['Profit'].isnull())]))+1)[::-1]

        append_data.append(df)

    combine_df = pd.concat(append_data)
    
    #combine_df.to_csv('single_df.csv')
    
combine_df['Release Date'] = pd.to_datetime(combine_df['Release Date'])

combine_df.loc[combine_df['Profitable Movie Director Count'] == 0, 'Profitable Movie Director Count'] = np.NaN
combine_df.loc[combine_df['Unprofitable Movie Director Count'] == 0, 'Unprofitable Movie Director Count'] = np.NaN
combine_df
combine_df.columns
combine_df['Director'].unique()
combine_df['Director'].unique().shape
df.columns
df = combine_df.loc[combine_df['Release Date'].dt.year < 2020] # 2020 is an outlier...

df = df.loc[df['Profit'].notnull()] # over 150 rows have incomplete information... 

df = df.loc[df['Role'] == 'Director']
print('did not direct a movie:', set(combine_df['Director'].unique()) - set(df['Director'].unique()))

quit_df = df.loc[df['Stopped Making Movies'] == True]
quit_df = quit_df.loc[quit_df['Director'] != '<NAME>']

stay_df = df.loc[df['Stopped Making Movies'] == False]

compare_df = df.loc[(df['Director'] == '<NAME>') | (df['Director'] =='<NAME>') 
                    | (df['Director'] == '<NAME>') | (df['Director'] == '<NAME>') 
                    | (df['Director'] == '<NAME>')]
sns.lineplot(x='Release Date', y='Profit', hue='Director', data=quit_df)
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.axhline(0, c='k', linestyle='--')
sns.lineplot(x='Release Date', y='Profit', hue='Director', data=compare_df)
plt.axhline(0, c='k', linestyle='--')
sns.lineplot(x='Personal Director Count', y='Profit', hue='Director', data=quit_df)
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.axhline(0, c='k', linestyle='--')
plt.axvline(3, c='r', linestyle='--')
sns.lineplot(x='Personal Director Count', y='Profit', hue='Director', data=compare_df)
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.axhline(0, c='k', linestyle='--')
plt.axvline(3, c='r', linestyle='--')
compare_df.loc[(compare_df['Profit'] < 0) & (compare_df['Personal Director Count'] <= 3)]['Director']
sns.lineplot(x='Release Date', y='Cumulated Profit', hue='Director', data=quit_df)
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
sns.scatterplot(x='Release Date', y='Unprofitable Movie Director Count', hue='Director', data=quit_df)
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
quit_df.groupby('Director')['Unprofitable Movie Director Count'].max()
stay_df.groupby('Director')['Unprofitable Movie Director Count'].max()
print(quit_df['Director'].unique())
print(len(quit_df['Director'].unique()))
df.drop('Stopped Making Movies',axis=1)<jupyter_output><empty_output><jupyter_text>sklearn practice<jupyter_code>features = df[['Personal Role Count',
       'Personal Director Count', 'Personal Movie count', 'Profit',
       'Cumulated Profit', 'Profitable Movie Count',
       'Unprofitable Movie Count', 'Unknown profit Movie Count',
       'Profitable Movie Director Count', 'Unprofitable Movie Director Count',
       'Unknown profit Movie Director Count']]
labels = df[['Stopped Making Movies']]

rf = RandomForestClassifier()

scores = cross_val_score(rf, features, labels.values.ravel(), cv=5)
print(scores)
def print_results(results):
    print('BEST PARAMS: {}\n'.format(results.best_params_))

    means = results.cv_results_['mean_test_score']
    stds = results.cv_results_['std_test_score']
    for mean, std, params in zip(means, stds, results.cv_results_['params']):
        print('{} (+/-{}) for {}'.format(round(mean, 3), round(std * 2, 3), params))
        
parameters = {
    'n_estimators': [5, 10, 20],
    'max_depth': [2, 10, 20, None]
}

cv = GridSearchCV(rf, parameters, cv=5)
cv.fit(features, labels.values.ravel())

print_results(cv)
x_train, x_test, y_train, y_test = train_test_split(features, labels, test_size=0.4, random_state=42)
x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=0.5, random_state=42)

rf = RandomForestClassifier(n_estimators=5)
rf.fit(x_train, y_train.values.ravel())

print(rf.score(x_test, y_test))
y_pred = rf.predict(x_test)
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True)
# x = df.drop('Stopped Making Movies',axis=1)
# y = df['Stopped Making Movies']
#x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.4, random_state=42)
from sklearn.neural_network import MLPClassifier
mlp = MLPClassifier(hidden_layer_sizes=(13,13,13),max_iter=500)
mlp.fit(x_train, y_train)
predictions = mlp.predict(x_test)
print(confusion_matrix(y_test,predictions))
print(classification_report(y_test,predictions))
stay_df
df
df.loc[df['Director'] == '<NAME>'].plot(x='Release Date', y='Profit')
sns.lineplot(x='Release Date', y='Profit', hue='Director', data=df)
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1692555408221254, 'fraction_numerical': 0.016480394014017807, 'mean_word_length': 3.3940907199334167, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 20, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '82904', 'license': 'MIT', 'n_tokens_mistral': 3776, 'n_tokens_neox': 3524, 'n_words': 774, 'path': 'rb/movie_director_loss/movie_director_boxoffice.ipynb', 'repo_name': 'joejoeyjoseph/playground', 'size': '10593'}