<jupyter_start><jupyter_text>Data Cleaning<jupyter_code>movies_clean = movies.copy()
# Drop columns that don't provide useful information: tagline, title, movie_id, title.1, overview, original_title, id, homepage
# ?Drop 'status' != Released before column drop?
movies_clean.drop(columns=['tagline', 'title', 'movie_id', 'title.1', 'overview', 'original_title', 'id', 'homepage'], inplace=True)
movies_clean.head(1)
# Look at Zero-budget/Zero-revenue movies
sum((movies_clean['budget']==0).apply(int) & (movies_clean['revenue']==0).apply(int))
# Drop observations with zero budget, less than $1000 revenue
movies_clean = movies_clean[movies_clean['budget']>0]
movies_clean = movies_clean[movies_clean['revenue']>1000]
movies_clean = movies_clean.reset_index(drop=True)
# Clean up genres column/remove empty's
print(movies_clean.shape)
print(movies_clean['genres'][movies_clean['genres'].apply(len)<3].count())
movies_clean = movies_clean[movies_clean['genres'].apply(len)>2]
print(movies_clean.shape)
# Clean up 'production_companies' column.  35 empty crew observation
print(movies_clean.shape)
print(movies_clean['production_companies'][movies_clean['production_companies'].apply(len)<3].count())
movies_clean = movies_clean[movies_clean['production_companies'].apply(len)>2]
print(movies_clean.shape)
# Clean up 'cast' column.  1 empty cast observations
print(movies_clean.shape)
print(movies_clean['cast'][movies_clean['cast'].apply(len)<3].count())
movies_clean = movies_clean[movies_clean['cast'].apply(len)>2]
print(movies_clean.shape)
# Clean up 'crew' column.  1 empty crew observation
print(movies_clean.shape)
print(movies_clean['crew'][movies_clean['crew'].apply(len)<3].count())
movies_clean = movies_clean[movies_clean['crew'].apply(len)>2]
print(movies_clean.shape)
# Clean up 'release_date' column.
print(movies_clean.shape)
print(movies_clean['release_date'][movies_clean['release_date'].apply(len)<3].count())
movies_clean = movies_clean[movies_clean['release_date'].apply(len)>1]
print(movies_clean.shape)

#movies_clean['release_date'] = movies_clean['release_date'].astype(int)
movies_clean = movies_clean.reset_index(drop=True)
# One-hot encode genres
#movies_clean['genre_names'] = movies_clean['genres'].apply(lambda s: '|'.join([e['name'] for e in json.loads(s)]))
#genre_one_hot = movies_clean['genre_names'].str.get_dummies('|')
#movies_clean_one_hot = pd.concat([movies_clean, genre_one_hot], axis=1)
movies_clean_one_hot = movies_clean.copy()
movies_clean_one_hot.head(2)
# Pull out all production_companies
production_companies_df = pd.DataFrame(columns=['production_companies'])
for row in movies_clean_one_hot.iterrows():
    for i in range(len(json.loads(row[1][5]))):   # row[1][5] is production_countries string
        production_companies_df = pd.concat([production_companies_df, pd.DataFrame([{'production_companies':json.loads(row[1][5])[i]['name']}])], ignore_index=True)
print(production_companies_df.shape)
production_companies_df.value_counts()
# Create dataframe from production_companies_df.value_counts()
production_companies_counts_df = pd.concat([production_companies_df.value_counts().index.get_level_values(0).to_frame(index=False), pd.DataFrame(production_companies_df.value_counts().values, columns=['count'])], axis=1)
# Create production_companies rank summation
comp_counts = movies_clean_one_hot['production_companies'].apply(lambda s: sum([int(production_companies_counts_df[production_companies_counts_df['production_companies']==comp['name']]['count']) for comp in json.loads(s)]))
comp_counts.rename('production_rank', inplace=True)
movies_clean_one_hot = pd.concat([movies_clean_one_hot, comp_counts], axis=1)
#movies_clean_one_hot.drop(['production_companies', 'production_companies_names'], axis=1, inplace=True)
movies_clean_one_hot.head(2)
movies_clean_one_hot.shape
# Create dataframe from .value_counts()
#cast_counts_df = pd.concat([cast_df.value_counts().index.get_level_values(0).to_frame(index=False), pd.DataFrame(cast_df.value_counts().values, columns=['count'])], axis=1)
#cast_counts_df
"""
cast_revenue_df = pd.DataFrame(columns=['name', 'revenue'])

for row in movies_clean_one_hot.iterrows():
    for i in range(len(json.loads(row[1][13]))):   # row[1][21] is cast string in ORIGINAL dataset
        cast_revenue_df = pd.concat([cast_revenue_df, pd.DataFrame([{'name':json.loads(row[1][13])[i]['name'], 'revenue':row[1][15]*math.exp(-0.4*i)}])], ignore_index=True)

cast_revenue_df.to_csv('/content/drive/MyDrive/Colab Notebooks/cast_revenue_budget.csv', index=False)
"""
cast_revenue_df = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/cast_revenue_budget.csv')
cast_revenue_df_unique = pd.unique(cast_revenue_df['name'])
cast_revenue_df_unique
"""
cast_cumm_revenue = pd.DataFrame(columns=['name', 'cumm_revenue'])

for name in cast_revenue_df_unique:
    cast_cumm_revenue = cast_cumm_revenue.append({'name': name, 'cumm_revenue': sum([row[1][1] for row in cast_revenue_df[cast_revenue_df['name'] == name].iterrows()])}, ignore_index=True)

#cast_cumm_revenue = cast_cumm_revenue.sort_values(by='cumm_revenue', ascending=False)
"""
#cast_cumm_revenue.to_csv('/content/drive/MyDrive/Colab Notebooks/cast_cumm_revenue_budget.csv', index=False)
cast_cumm_revenue = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/cast_cumm_revenue_budget.csv')
# Concat a 'cast_worth' column to movies, which comprises a weighted-sum of all cast member's cummulative worth
"""
cast_worth = pd.DataFrame(columns=['cast_worth'])
for row in movies_clean_one_hot.iterrows():
    cast_worth = pd.concat([cast_worth, pd.DataFrame({'cast_worth': sum([int(cast_cumm_revenue[cast_cumm_revenue['name']==e['name']]['cumm_revenue'])*math.exp(-0.4*i) for i,e in enumerate(json.loads(row[1][13]))])}, index=[0])], ignore_index=True)
"""
#cast_worth.to_csv('/content/drive/MyDrive/Colab Notebooks/cast_worth.csv', index=False)
cast_worth = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/cast_worth.csv')
movies_clean_one_hot['cast_worth'] = cast_worth
# Add director column
"""
director = []
for row in movies_clean.iterrows():
    for e in json.loads(row[1][14]):
        if e['job']=='Director':
            director.append(e['name'])
            break
"""
#movies_clean_one_hot = pd.concat([movies_clean_one_hot, pd.DataFrame(director, columns=['director'])], axis=1)
movies_clean_one_hot['release_date'] = movies_clean_one_hot['release_date'].str.slice(start=5, stop=7).apply(int)
#movies_clean['release_date'] = movies_clean['release_date'].astype(int)
movies_clean_one_hot.head()<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis<jupyter_code># Popularity histogram
plt.hist(movies_clean_one_hot['popularity'], bins=50);
movies_clean_one_hot['vote_average'].hist();
movies_clean_one_hot.plot(kind='scatter', x='budget', y='revenue', title='stuff vs. stuff');
# vote_count
movies_clean_one_hot['vote_count'].hist();
movies_clean_one_hot['cast_worth'].hist();
movies_clean_one_hot.plot(kind='scatter', loglog=True, x='cast_worth', y='revenue', title='stuff vs. stuff');
movies_clean_one_hot.corr()['revenue']<jupyter_output><empty_output><jupyter_text>Data Processing<jupyter_code>y = movies_clean_one_hot['revenue']
X = movies_clean_one_hot.drop('revenue', axis=1).copy()
#X.drop(['genres','keywords','status','original_language','production_companies','production_countries','spoken_languages','cast','crew','genre_names'], axis=1, inplace=True)
X.drop(['genres','keywords','status','original_language','production_companies','production_countries','spoken_languages','cast','crew'], axis=1, inplace=True)
X.head()<jupyter_output><empty_output><jupyter_text>Decision Trees<jupyter_code>numLoops = 100
depth_vec = np.arange(1,11)
depth_error = np.zeros(len(depth_vec))

for outIdx in range(0,len(depth_error)):
  mean_error = np.zeros(numLoops)
  depth_val  = outIdx + 1
  model = DecisionTreeRegressor(max_depth=depth_val)
  for idx in range(0,numLoops):
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    mean_error[idx] = rmspe(y_test, y_pred)

  depth_error[outIdx] = np.mean(mean_error)

depth_error
plt.plot(depth_error);<jupyter_output><empty_output><jupyter_text>Decision Tree Graph<jupyter_code>model = DecisionTreeRegressor(max_depth=3, criterion='mae')
model.fit(X,y)
import graphviz
import pydotplus
from IPython.display import display
from sklearn import tree

display(graphviz.Source(tree.export_graphviz(model, feature_names=X.columns)));<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>numTrees = 20

rmse_all = np.zeros(numTrees)

for outIdx in range(0,numTrees):

  numLoops = 20
  rmse_inside = np.zeros(numLoops)

  for inIdx in range(0,numLoops):

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
    rfModel = RandomForestRegressor(n_estimators = (outIdx+1), max_depth=4)
    rfModel.fit(X_train, y_train)
    y_pred_rf = rfModel.predict(X_test)
    rmse_inside[inIdx] = root_mean_squared_percentage_error(y_test, y_pred_rf, 1e6)

  #print(outIdx,' trees finished.')
  rmse_all[outIdx] = np.mean(rmse_inside)

print(rmse_all)

plt.plot(rmse_all);<jupyter_output>[620.58320017 635.17325615 561.55904365 560.66516669 599.25409158
 549.82133987 620.98341717 553.41249321 576.21220601 607.00702501
 569.58732482 591.78915279 580.93628101 589.78547018 571.59883378
 586.24518045 578.6423132  556.44891043 567.65653475 551.95085959]<jupyter_text>XGBoost<jupyter_code>numLoops = 100
mse_xgb = np.zeros(numLoops)

for idx in range(0,numLoops):
  X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
  xgbr = xgb.XGBRegressor(objective = 'reg:squarederror')
  xgbr.fit(X_train, np.log10(y_train))
  y_pred_xgb = xgbr.predict(X_test)
  mse_xgb[idx] = root_mean_squared_percentage_error(y_test, 10**y_pred_xgb, 1e6)

mse_xgb.mean()
np.sqrt(mse_xgb.mean())
plt.scatter(np.log10(y_test),((y_test - 10**y_pred_xgb)/(y_test))*100)
plt.ylim([-1000, 1000])
#plt.xlim([0,1e9])
feat_imp = pd.Series(xgbr.feature_importances_, index=X.columns)
feat_imp.plot(kind='bar', title='Feature Importance')
plt.ylabel('Feature Importance Score');
print('Done')<jupyter_output>Done
{'fraction_non_alphanumeric': 0.14976825028968713, 'fraction_numerical': 0.033700270374662035, 'mean_word_length': 7.920757967269595, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '556271', 'license': 'MIT', 'n_tokens_mistral': 4160, 'n_tokens_neox': 3883, 'n_words': 527, 'path': 'Project_3.ipynb', 'repo_name': 'kengustafson/DeepDive-DataScience', 'size': '10356'}