<jupyter_start><jupyter_text>Model Dependencies & AWS Connection<jupyter_code>#importing libraries
from sqlalchemy import create_engine
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from collections import Counter
import pandas as pd
from sklearn import linear_model
import statsmodels.api as sm
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn import metrics
from sklearn.model_selection import train_test_split, KFold, cross_val_score
from sklearn.linear_model import LinearRegression, Ridge, Lasso, LassoCV, Ridge, RidgeCV
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error
import scipy.stats as stats
# Create connection to AWS table
engine = create_engine('postgresql+psycopg2://postgres:<EMAIL>@<EMAIL>:5432/postgres')
# Create dataframe for main data to be used for modeling using connection with AWS
main_df = pd.read_sql_table('consolidated_pre_transformation', con=engine)
raw_main_df = pd.read_sql_table('consolidated_pre_transformation', con=engine)<jupyter_output><empty_output><jupyter_text>Feature Transformation<jupyter_code># Preview main dataframe
main_df.head()

#Split out genre into columns (manual dummy variable creation)

main_df['isHorror'] = main_df['genre'].apply(lambda x: 1 if 'Horror' in x else 0)
main_df['isComedy'] = main_df['genre'].apply(lambda x: 1 if 'Comedy' in x else 0)
main_df['isRomance'] = main_df['genre'].apply(lambda x: 1 if 'Romance' in x else 0)
main_df['isFantasy'] = main_df['genre'].apply(lambda x: 1 if 'Fantasy' in x else 0)
main_df['isWestern'] = main_df['genre'].apply(lambda x: 1 if 'Western' in x else 0)
main_df['isWar'] = main_df['genre'].apply(lambda x: 1 if 'War' in x else 0)
main_df['isHistory'] = main_df['genre'].apply(lambda x: 1 if 'History' in x else 0)
main_df['isDrama'] = main_df['genre'].apply(lambda x: 1 if 'Drama' in x else 0)
main_df['isSport'] = main_df['genre'].apply(lambda x: 1 if 'Sport' in x else 0)
main_df['isMusic'] = main_df['genre'].apply(lambda x: 1 if 'Music' in x else 0)
main_df['isMusical'] = main_df['genre'].apply(lambda x: 1 if 'Musical' in x else 0)
main_df['isAnimation'] = main_df['genre'].apply(lambda x: 1 if 'Animation' in x else 0)
main_df['isBiography'] = main_df['genre'].apply(lambda x: 1 if 'Biography' in x else 0)
main_df['isFamily'] = main_df['genre'].apply(lambda x: 1 if 'Family' in x else 0)
main_df['isSci-Fi'] = main_df['genre'].apply(lambda x: 1 if 'Sci-Fi' in x else 0)
main_df['isMystery'] = main_df['genre'].apply(lambda x: 1 if 'Mystery' in x else 0)
main_df['isAdventure'] = main_df['genre'].apply(lambda x: 1 if 'Adventure' in x else 0)
main_df['isCrime'] = main_df['genre'].apply(lambda x: 1 if 'Crime' in x else 0)
main_df['isThirller'] = main_df['genre'].apply(lambda x: 1 if 'Thriller' in x else 0)
main_df['isAction'] = main_df['genre'].apply(lambda x: 1 if 'Action' in x else 0)
#Create dummy variables for MPAA rating
mpaa_dummies_df = pd.get_dummies(main_df['mpa_rating'])
mpaa_dummies_df
#Add MPAA rating dummy variables to main dataframe
dummies_main_df = pd.concat([main_df, mpaa_dummies_df] , axis = 1)
dummies_main_df<jupyter_output><empty_output><jupyter_text>Final Data Cleansing<jupyter_code>#Remove N/A rows
clean_main_df = dummies_main_df.dropna()
# Create a new DataFrame that holds only the title names.
df_titles = clean_main_df.filter(["original_title"], axis=1)
# Create final clean dataframe while removing unecessary columns
final_main_df = clean_main_df.drop([ 'original_title', 'votes', 'genre', 'mpa_rating', ], axis=1)<jupyter_output><empty_output><jupyter_text>Review Feature Correlation<jupyter_code># Plot the Correlation map to see how features are correlated with target: SalePrice
corr_main_df = final_main_df.drop([ 'title_length', 'isWestern', 'isWar', 'isHistory' , 'NC-17', 'Unrated' , 'isSport' , 'isMusical', 'isMusic', 'month_number', 'is_oscar_directed' , 'isFantasy'], axis=1)
dfCorr = corr_main_df.corr()
filteredDf = dfCorr[((dfCorr >= .20) | (dfCorr <= -.20)) & (dfCorr !=1.000)]
plt.figure(figsize=(40,20))
sns.heatmap(filteredDf, annot=True, cmap="Reds")
plt.show()
# Save final clean machine learning dataset off to resources folder
resources_folder = r'../resources/'
final_main_df.to_csv(resources_folder + 'clean_ml_data.csv', index=False)
final_main_df.shape<jupyter_output><empty_output><jupyter_text>Ridge Regression Machine Learning Model<jupyter_code># Create X and y variables
X = final_main_df.drop(['imdb_rating' ], axis=1)
y = final_main_df['imdb_rating']
# Train, Test, & Split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 42)
# Rescale data
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
# Initiate ride regression model and fit model
rr = Ridge(alpha=10)
rr.fit(X_train_scaled,y_train)
# Generate predicted values
y_pred_rr = rr.predict(X_test_scaled)<jupyter_output><empty_output><jupyter_text>Review of Ridge Model Results<jupyter_code># Create dataframe with title, actual value, and predicted value

rr_diff = pd.DataFrame({'Actual value': y_test, 'Predicted value': y_pred_rr})
rr_diff['title'] = df_titles['original_title']
rr_diff.head(10)
#Create actual vs prediction scatter plot

plt.scatter(y_pred_rr, y_test)
plt.plot(np.linspace(4,10), np.linspace(4,10))
plt.title('Ridge Model Prediction vs. Target')
plt.xlabel('Predicted IMDb Rating')
plt.ylabel('Actual IMDb Rating')
# Function for creating diagnostic plots

def diagnostic_plot(X, y):
    plt.figure(figsize=(20,5))

    pred = X

    plt.subplot(1, 3, 1)
    plt.scatter(X,y)
    plt.plot(X, pred, color='blue',linewidth=1)
    plt.title("Regression fit")
    plt.xlabel("x")
    plt.ylabel("y")
    
    plt.subplot(1, 3, 2)
    res = y - pred
    plt.scatter(pred, res)
    plt.title("Residual plot")
    plt.xlabel("prediction")
    plt.ylabel("residuals")
    
    plt.subplot(1, 3, 3)
    #Generates a probability plot of sample data against the quantiles of a 
    # specified theoretical distribution 
    stats.probplot(res, dist="norm", plot=plt)
    plt.title("Normal Q-Q plot");

diagnostic_plot(y_pred_rr, y_test)
#Using RidgeCV to determine idea Alpha

m_alpha = RidgeCV(alphas=[0.1, 1.0, 10.0])
model_cv = m_alpha.fit(X_train_scaled, y_train)
print('Ideal Alpha: {}'.format(model_cv.alpha_))
#Using RidgeCV to determine idea Alpha using a more precise range

m_alpha = RidgeCV(alphas=[8.0, 8.5, 9.0, 9.5, 10.0])
model_cv = m_alpha.fit(X_train_scaled, y_train)
print('Ideal Alpha: {}'.format(model_cv.alpha_))
#Print ridge regression score statistics

from sklearn.metrics import mean_absolute_error
r2 = r2_score(y_test, y_pred_rr)
mean_abs_error = mean_absolute_error(y_test, y_pred_rr)
mse = mean_squared_error(y_test,y_pred_rr)
accuracy = 100- (np.mean(np.abs((y_test - y_pred_rr) / y_test)) * 100)

print('R Squared: {:.3f}'.format(r2))
print('MAE: {:.3f}'.format(mean_abs_error))
print('MSE: {:.3f}'.format(mse))
print('RMSE: {:.3f}'.format(np.sqrt(mse)))
print('Accuracy: {:.2f}'.format(100- (np.mean(np.abs((y_test - y_pred_rr) / y_test)) * 100)))<jupyter_output>R Squared: 0.322
MAE: 0.596
MSE: 0.613
RMSE: 0.783
Accuracy: 89.46<jupyter_text>Create and Analyze Actual vs Predicted Dataframe<jupyter_code># Create actual vs predicted column
rr_diff['res'] = rr_diff['Predicted value'] - rr_diff['Actual value']
rr_diff.head()
#Sort and rename columns
act_vs_pred_df = rr_diff.reindex(columns= ['title', 'Actual value' , 'Predicted value' , 'res'])
comp_df = act_vs_pred_df.rename(columns={'title' : 'Title', 'Actual value' : 'Actual Rating' , 'Predicted value' : 'Predicted Rating' , 'res' : 'Difference' })
#Merge actual vs predicted IMDB ratings and raw_main_dataframe for reference
comp_complete = pd.merge(comp_df, raw_main_df, how="left", left_on=['Title'],right_on=['original_title'])
#Sort by difference
comp_complete.sort_values(by=['Difference']).head(20)
#Add accuracy column
comp_complete["Accuracy"] = 100-(np.abs((comp_complete['Actual Rating'] - comp_complete['Predicted Rating'] ) / comp_complete['Actual Rating'])*100)
comp_complete.head()
# Move accuracy column
Acc_Col = comp_complete.pop('Accuracy')
comp_complete.insert(4, 'Accuracy' , Acc_Col)

comp_results = comp_complete.drop([ 'original_title' , 'imdb_rating'], axis=1)
clean_comp_results = comp_results.rename(columns={'votes' : 'Votes', 'title_length' : 'Title Length' , 'month_number' : 'Month', 'genre' : 'Genre' , 'duration' : 'Duration (min)' , 'budget_millions' : 'Budget (mm)' , 'oscar_nominated_actors_count' : 'Oscar Nominated Actors' , 'mpa_rating' : 'MPAA Rating'})
#Round column data for better display amounts
clean_comp_results = clean_comp_results.round({"Predicted Rating":2, "Difference":2, "Accuracy":2})
# Create interactive actual vs prediction scatter plot
import plotly.express as px

df = clean_comp_results
fig = px.scatter(df, x="Predicted Rating", y="Actual Rating", trendline="ols", hover_data=['Accuracy','Title', 'Genre' , 'Duration (min)' , 'Budget (mm)' , 'MPAA Rating' ])
fig.show()
# Display Most Accurate Top 10

most_accurate = clean_comp_results.sort_values(by=['Accuracy']).tail(10)
most_accurate
# Display Least Accurate Bottom 10
least_accurate = clean_comp_results.sort_values(by=['Accuracy']).head(10)
least_accurate
resources_folder = r'../resources/'
clean_comp_results.to_csv(resources_folder + 'ridge_test_results.csv', index=False)<jupyter_output><empty_output><jupyter_text>Review features for most important<jupyter_code># Combine coefficients and column names
coef_zip = list(zip(final_main_df.drop(['imdb_rating' ], axis=1).columns.tolist(), rr.coef_))
sorted_coef = sorted(coef_zip, key = lambda x: abs(x[1]))
# Sort list
sorted_coef[:-10:-1]
# Setting up the parameters to plot the most important features
plot_name = [name[0].title() for name in sorted_coef[:-10:-1]]
plot_coef = [coef[1] for coef in sorted_coef[:-10:-1]]
colors = [0 if c>=0 else 1 for c in plot_coef]
# Setting up the parameters to plot the most important features
sns.set_theme(style="whitegrid")
cmap = sns.diverging_palette(240, 10, n=2)
movie_coef = sns.barplot(
    y=plot_name,
    x=plot_coef,
    hue=colors,
    palette=cmap,
    orient='h',
    )
movie_coef.set_title('Top 10 Important Features for IMDb Rating',fontsize=20,y=1.05, x=0.31)
movie_coef.get_legend().set_visible(False)
movie_coef.set_xlabel('Coefficient',fontsize=15)
movie_coef.set_ylabel('Feature',fontsize=18)
plt.xticks(fontsize=15)
plt.yticks(fontsize=15)
plt.show()
fig = movie_coef.get_figure()
# fig.savefig('top_features.png', bbox_inches='tight');<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13659078289838977, 'fraction_numerical': 0.019526189154173607, 'mean_word_length': 6.59451862262825, 'pattern_counts': {'":': 3, '<': 35, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '553863', 'license': 'MIT', 'n_tokens_mistral': 4110, 'n_tokens_neox': 3894, 'n_words': 950, 'path': 'Jupyter_Notebook_Files/Predict_IMDB_Ratings_RidgeRegression.ipynb', 'repo_name': 'danielbrock4/Ratings_On_Demand', 'size': '10864'}