<jupyter_start><jupyter_text>baseline model  - linear regression numerical features<jupyter_code>movie_df1 = movie_df.loc[:, ['yrs_since_release', 'runtime', 'votes', 'gross', 'rating']]

X = movie_df1.drop('rating', axis = 1)
X_ols = sm.add_constant(X)
y = movie_df1['rating']

model_n = sm.OLS(y, X_ols)
fit_n = model_n.fit()
fit_n.summary()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=50)

lr_n = LinearRegression()
lr_n.fit(X_train, y_train)
y_pred_train_1 = lr_n.predict(X_train)
print("R2 train: ", lr_n.score(X_train, y_train))
print("RMSE train: ", mean_squared_error(y_train, y_pred_train_1))

y_pred_test_1 = lr_n.predict(X_test)
print("R2 test: ", lr_n.score(X_test, y_test))
print("RMSE test: ", mean_squared_error(y_test, y_pred_test_1))<jupyter_output>R2 train:  0.3558155509231118
RMSE train:  0.4929969059595019
R2 test:  0.3501006750673985
RMSE test:  0.5847677139817105<jupyter_text>numerical and categorical<jupyter_code>X2 = movie_df.drop('rating', axis=1)
X2_ols = sm.add_constant(X2)
y2 = movie_df['rating']

model_nc = sm.OLS(y2, X2_ols)
fit_nc = model_nc.fit()
fit_nc.summary()
X2_train, X2_test, y2_train, y2_test = train_test_split(X2, y2, test_size=0.3, random_state=50)

lr_2 = LinearRegression()
lr_2.fit(X2_train, y2_train)
y_pred_train_2 = lr_2.predict(X2_train)
print('R2 train: ', lr_2.score(X2_train, y2_train))
print('RMSE train: ', mean_squared_error(y2_train, y_pred_train_2))

y_pred_test_2 = lr_2.predict(X2_test)
print('R2 test: ', lr_2.score(X2_test, y2_test))
print('RMSE test: ', mean_squared_error(y2_test, y_pred_test_2))<jupyter_output>R2 train:  0.517954468203345
RMSE train:  0.3689113514737899
R2 test:  0.4958687076035617
RMSE test:  0.45360826222104933<jupyter_text>numerical and categorical - subst  not including: History, <NAME>, <NAME>, <NAME>, Mystery, Thriller, <NAME>,  <NAME><jupyter_code>movie_df.columns
movie_df3 = movie_df.loc[:, ['runtime', 'rating', 'votes', 'gross', 'yrs_since_release', 'certificate_PG-13', 
                             'certificate_R', 'Action', 'Adventure', 'Animation', 'Biography', 'Comedy', 
                             'Crime', 'Documentary', 'Drama', 'Family', 'Fantasy', 'Film-Noir', 'Horror', 
                             'Music', 'Musical', 'News', 'Romance', 'Sci-Fi', 'Sport', 'War', 
                             'Western',  '<NAME>', '<NAME>', '<NAME>', '<NAME>', 
                             '<NAME>', '<NAME>', '<NAME>', '<NAME>', 
                             '<NAME>', '<NAME>', '<NAME>', '<NAME>', 
                             '<NAME>', '<NAME>', '<NAME>']]

X3 = movie_df3.drop('rating', axis = 1)
X3_ols = sm.add_constant(X3)
y3 = movie_df3['rating']

model_3 = sm.OLS(y3, X3_ols)
fit_3 = model_3.fit()
fit_3.summary()
X3_train, X3_test, y3_train, y3_test = train_test_split(X3, y3, test_size=0.3, random_state=50)

lr_3 = LinearRegression()
lr_3.fit(X3_train, y3_train)
y_pred_train_3 = lr_3.predict(X3_train)
print('R2 train: ', lr_3.score(X3_train, y3_train))
print('RMSE train: ', mean_squared_error(y3_train, y_pred_train_3))

y_pred_test_3 = lr_3.predict(X3_test)
print('R2 test: ', lr_3.score(X3_test, y3_test))
print('RMSE test: ', mean_squared_error(y3_test, y_pred_test_3))<jupyter_output>R2 train:  0.5175736255774701
RMSE train:  0.3692028118412093
R2 test:  0.49735701344811356
RMSE test:  0.4522691114125538<jupyter_text>numerical and categorical - another subset<jupyter_code>movie_df4 = movie_df.loc[:, ['runtime', 'rating', 'votes', 'gross', 'yrs_since_release', 'certificate_PG-13', 
                             'certificate_R', 'Action', 'Adventure', 'Animation', 'Biography', 'Comedy', 
                             'Crime', 'Documentary', 'Drama', 'Family', 'Fantasy', 'Film-Noir', 'Horror', 
                             'Music', 'Musical', 'News', 'Romance', 'Sci-Fi', 'War', 'Western',  
                             '<NAME>', '<NAME>', '<NAME>', '<NAME>', 
                             '<NAME>', '<NAME>', '<NAME>', '<NAME>', 
                             '<NAME>', '<NAME>', '<NAME>', '<NAME>', 
                             '<NAME>']]

X4 = movie_df4.drop('rating', axis = 1)
X4_ols = sm.add_constant(X4)
y4 = movie_df4['rating']

model_4 = sm.OLS(y4, X4_ols)
fit_4 = model_4.fit()
fit_4.summary()
X4_train, X4_test, y4_train, y4_test = train_test_split(X4, y4, test_size=0.3, random_state=50)

lr_4 = LinearRegression()
lr_4.fit(X4_train, y4_train)
y_pred_train_4 = lr_4.predict(X4_train)
print('R2 train: ', lr_4.score(X4_train, y4_train))
print('RMSE train: ', mean_squared_error(y4_train, y_pred_train_4))

y_pred_test_4 = lr_4.predict(X4_test)
print('R2 test: ', lr_4.score(X4_test, y4_test))
print('RMSE test: ', mean_squared_error(y4_test, y_pred_test_4))<jupyter_output>R2 train:  0.5173975962217147
RMSE train:  0.36933752780318235
R2 test:  0.497584146481299
RMSE test:  0.4520647411978297<jupyter_text>baseline vs other models<jupyter_code>df_columns = ['runtime', 'rating', 'votes', 'gross', 'yrs_since_release', 'certificate_PG-13', 
              'certificate_R', 'Action', 'Adventure', 'Animation', 'Biography', 'Comedy', 'Crime', 
              'Documentary', 'Drama', 'Family', 'Fantasy', 'Film-Noir', 'Horror', 'Music', 'Musical', 
              'News', 'Romance', 'Sci-Fi', 'War', 'Western', '<NAME>', '<NAME>', 
              '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', 
              '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', 
              '<NAME>']

model_df = movie_df.loc[:, df_columns]
X = model_df.drop('rating', axis=1)
y = model_df.rating

# split into sets
X, X_test, y, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# split training data into multiple folds
kf = KFold(n_splits=5, shuffle=True, random_state=0)

# cross validation starts
X, y = np.array(X), np.array(y)
#collect the validation results for all models
cv_lm_r2s, cv_lm_rid_r2s, cv_lm_lasso_r2s, cv_lm_poly_r2s = [], [], [], [] 

cv_lm_r2s_tr, cv_lm_rid_r2s_tr, cv_lm_lasso_r2s_tr, cv_lm_poly_r2s_tr = [], [], [], []

for train_ind, val_ind in kf.split(X,y):
    
    X_train, y_train = X[train_ind], y[train_ind]
    X_val, y_val = X[val_ind], y[val_ind] 
    
    # models
    lm = LinearRegression()
    lm_rid = Ridge(alpha=1)
    lm_lasso = Lasso(alpha=0.005)
    lm_poly = LinearRegression()
    
    # scaling for train, val, test 
    scaler = StandardScaler()
    X_train_scaled = scaler.fit_transform(X_train)
    X_val_scaled = scaler.transform(X_val)

    poly = PolynomialFeatures(degree=2)
    X_train_poly = poly.fit_transform(X_train)
    X_val_poly = poly.transform(X_val)
    
    # linear
    lm.fit(X_train, y_train)
    cv_lm_r2s.append(lm.score(X_val, y_val))
    cv_lm_r2s_tr.append(lm.score(X_train, y_train))    
    lm_train_preds = lm.predict(X_train)
    linear_train_rmse = mean_squared_error(y_train, lm_train_preds)
    linear_train_mae = mean_absolute_error(y_train, lm_train_preds)
    
    lm_val_preds = lm.predict(X_val)
    linear_val_rmse = mean_squared_error(y_val, lm_val_preds)
    linear_val_mae = mean_absolute_error(y_val, lm_val_preds)
    
    
    # ridge
    lm_rid.fit(X_train_scaled, y_train)
    cv_lm_rid_r2s.append(lm_rid.score(X_val_scaled, y_val))
    cv_lm_rid_r2s_tr.append(lm_rid.score(X_train_scaled, y_train))
    
    rid_train_preds = lm_rid.predict(X_train_scaled)
    ridge_train_rmse = mean_squared_error(y_train, rid_train_preds)
    ridge_train_mae = mean_absolute_error(y_train, rid_train_preds)
    
    rid_val_preds = lm_rid.predict(X_val_scaled)
    ridge_val_rmse = mean_squared_error(y_val, rid_val_preds)
    ridge_val_mae = mean_absolute_error(y_val, rid_val_preds)
    
    # lasso
    lm_lasso.fit(X_train_scaled, y_train)
    cv_lm_lasso_r2s.append(lm_lasso.score(X_val_scaled, y_val))
    cv_lm_lasso_r2s_tr.append(lm_lasso.score(X_train_scaled, y_train))
    
    lasso_train_preds = lm_lasso.predict(X_train_scaled)
    lasso_train_rmse = mean_squared_error(y_train, lasso_train_preds)
    lasso_train_mae = mean_absolute_error(y_train, lasso_train_preds)
    
    lasso_val_preds = lm_lasso.predict(X_val_scaled)
    lasso_val_rmse = mean_squared_error(y_val, lasso_val_preds)
    lasso_val_mae = mean_absolute_error(y_val, lasso_val_preds)
        
    # poly model
    lm_poly.fit(X_train_poly, y_train)
    cv_lm_poly_r2s.append(lm_poly.score(X_val_poly, y_val))
    cv_lm_poly_r2s_tr.append(lm_poly.score(X_train_poly, y_train))
    
    poly_train_preds = lm_poly.predict(X_train_poly)
    poly_train_rmse = mean_squared_error(y_train, poly_train_preds)
    poly_train_mae = mean_absolute_error(y_train, poly_train_preds)
    
    poly_val_preds = lm_poly.predict(X_val_poly)
    poly_val_rmse = mean_squared_error(y_val, poly_val_preds)
    poly_val_mae = mean_absolute_error(y_val, poly_val_preds)
    

print('Training set: ', '\n')
print('simple regression scores: ', cv_lm_r2s_tr)
print('simple RMSE: ', linear_train_rmse)
print('simple MAE: ', linear_train_mae, '\n')

print('ridge scores: ', cv_lm_rid_r2s_tr)
print('ridge RMSE: ', ridge_train_rmse)
print('ridge MAE: ', ridge_train_mae, '\n')

print('lasso scores: ', cv_lm_lasso_r2s_tr)
print('lasso RMSE: ', lasso_train_rmse)
print('lasso MAE: ', lasso_train_mae, '\n')

print('poly scores: ', cv_lm_poly_r2s_tr)
print('poly RMSE: ', poly_train_rmse)
print('poly MAE: ', poly_train_mae, '\n')

print('Val set: ', '\n')
print('simple regression scores: ', cv_lm_r2s)
print('simple RMSE: ', linear_val_rmse)
print('simple MAE: ', linear_val_mae, '\n')

print('ridge scores: ', cv_lm_rid_r2s)
print('ridge RMSE: ', ridge_val_rmse)
print('ridge MAE: ', ridge_val_mae, '\n')

print('lasso scores: ', cv_lm_lasso_r2s)
print('lasso RMSE: ', lasso_val_rmse)
print('lasso MAE: ', lasso_val_mae, '\n')

print('Poly scores: ', cv_lm_poly_r2s)
print('Poly RMSE: ', poly_val_rmse)
print('Poly MAE: ', poly_val_mae, '\n')<jupyter_output>Training set:  

simple regression scores:  [0.5335569447649865, 0.518258293084872, 0.5278552741380831, 0.5216061071566196, 0.5150354589192558]
simple RMSE:  0.3865102764146871
simple MAE:  0.4592352307408031 

ridge scores:  [0.533556744478888, 0.5182580966763837, 0.5278550872490984, 0.5216059099380028, 0.5150352647745224]
ridge RMSE:  0.3865104311454524
ridge MAE:  0.45924452947776606 

lasso scores:  [0.5320276518574737, 0.5168940571047947, 0.5263504607124367, 0.5202209493135503, 0.5135507251172258]
lasso RMSE:  0.38769358946876326
lasso MAE:  0.45991226460792256 

poly scores:  [0.4708342871020732, 0.6817578616103779, 0.680737122962694, 0.6766362834579978, 0.5791091957091521]
poly RMSE:  0.3354443620647523
poly MAE:  0.4301773658864211 

Val set:  

simple regression scores:  [0.4714950761329443, 0.52809509908417, 0.48557981694820596, 0.5144022966533345, 0.5410859445212699]
simple RMSE:  0.3514994118014472
simple MAE:  0.4197072274952082 

ridge scores:  [0.47145830246556164, 0.528[...]<jupyter_text>test<jupyter_code>#test - linear
lm.fit(X, y)
lm_preds = lm.predict(X_test)

linear_test_rmse = mean_squared_error(y_test, lm_preds, squared=False)
linear_test_mae = mean_absolute_error(y_test, lm_preds)
linear_test_r2 = lm.score(X_test, y_test)

#test - Ridge
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
X_test_scaled = scaler.transform(X_test)
    
lm_rid.fit(X_scaled, y)
ridge_preds = lm_rid.predict(X_test_scaled)

ridge_test_rmse = mean_squared_error(y_test, ridge_preds, squared=False)
ridge_test_mae = mean_absolute_error(y_test, ridge_preds)
ridge_test_r2 = lm_rid.score(X_test_scaled, y_test)

#test - Lasso
lm_lasso.fit(X_scaled, y)
lasso_preds = lm_lasso.predict(X_test_scaled)

lasso_test_rmse = mean_squared_error(y_test, lasso_preds, squared=False)
lasso_test_mae = mean_absolute_error(y_test, lasso_preds)
lasso_test_r2 = lm_lasso.score(X_test_scaled, y_test)

#test - poly
poly = PolynomialFeatures(degree=2)
X_poly = poly.fit_transform(X)
X_test_poly = poly.transform(X_test)

lm_poly.fit(X_poly, y)
poly_preds = lm_poly.predict(X_test_poly)

poly_test_rmse = mean_squared_error(y_test, poly_preds, squared=False)
poly_test_mae = mean_absolute_error(y_test, poly_preds)
poly_test_r2 = lm_poly.score(X_test_poly, y_test)

column_list = ['Model_Type', 'R-squared', 'RMSE', 'MAE']

linear_dict = dict(zip(column_list, ['Simple Linear', linear_test_r2, linear_test_rmse, linear_test_mae]))
ridge_dict = dict(zip(column_list, ['Ridge', ridge_test_r2, ridge_test_rmse, ridge_test_mae]))
lasso_dict = dict(zip(column_list, ['LASSO', lasso_test_r2, lasso_test_rmse, lasso_test_mae]))
poly_dict = dict(zip(column_list, ['Polynomial', poly_test_r2, poly_test_rmse, poly_test_mae]))

score_df = pd.DataFrame([linear_dict, ridge_dict, lasso_dict, poly_dict])
score_df
plt.figure(figsize=(8,5))
plt.scatter(lm_preds, y_test, alpha=0.2)
plt.plot([0,10], [0,10])
plt.title('IMDB Ratings vs. Predicted Ratings Baseline', fontsize=14)
plt.xticks(np.arange(0, 10, step=1))
plt.xlabel('Predicted Ratings', fontsize=12)
plt.ylabel('IMDB Rating', fontsize=12)
plt.savefig('Baseline.png');<jupyter_output><empty_output><jupyter_text>Lasso has a high r-squared value and low MAE<jupyter_code>plt.figure(figsize=(8,4))
plt.scatter(lasso_preds, y_test, alpha=0.2)
plt.plot([0,10], [0,10])
plt.title('IMDB Ratings vs. Predicted Ratings', fontsize=14)
plt.xlabel('Predicted Ratings', fontsize=12)
plt.ylabel('IMDB Ratingd', fontsize=12)
plt.savefig('IMDB Final model.png')
residuals = y_test - lasso_preds
plt.figure(figsize=(8, 5))
plt.scatter(lasso_preds, residuals, alpha=0.2)
plt.plot([0,10], [0,0])
plt.title("Residuals vs. Predictions (Baseline)")
plt.xlabel("Prediction", size=12)
plt.ylabel("Residuals", size=12)
plt.savefig('residual_finalmodel.png');<jupyter_output><empty_output><jupyter_text>diagnostic_plot check<jupyter_code>def diagnostic_plot(x, y):
    plt.figure(figsize=(20,5))
    
    rgr = LinearRegression()
    rgr.fit(x,y)
    pred = rgr.predict(x)

    plt.subplot(1, 3, 1)
    plt.scatter(pred,y)
#     plt.plot([0,4.5],[0,4.5])
    plt.title("Regression fit")
    plt.xlabel("x")
    plt.ylabel("y")
    
    plt.subplot(1, 3, 2)
    res = y - pred
    plt.scatter(pred, res)
    plt.plot([0,10],[0,0])
    plt.title("Residual plot")
    plt.xlabel("prediction")
    plt.ylabel("residuals")
    
    plt.subplot(1, 3, 3)
    stats.probplot(res, dist="norm", plot=plt)
    plt.title("Normal Q-Q plot") #quantile to quantile plot 
x = model_df.drop('rating', axis=1)
y = model_df.rating

diagnostic_plot(X, y)
plt.savefig('diagnostic_plots.png');<jupyter_output><empty_output><jupyter_text>Lasso Features Analysis<jupyter_code># Lasso Feature Selection

XL = model_df.drop('rating', axis=1)
yL = model_df.rating

reg = LassoCV()
reg.fit(XL, yL)
print("Best alpha using built-in LassoCV: %f" % reg.alpha_)
print("Best score using built-in LassoCV: %f" %reg.score(XL,yL))

coef = pd.Series(reg.coef_ , index=XL.columns)

print("Lasso picked " + str(sum(coef != 0)) + " variables and eliminated the other " +  str(sum(coef == 0)) + " variables")
noz_coef = coef #[coef != 0]
lmp_coef = noz_coef.sort_values()
neg_coef = abs(lmp_coef[lmp_coef < 0]).sort_values()
pos_coef = lmp_coef[lmp_coef > 0]
plt.rcParams['figure.figsize'] = (20.0, 10.0)
lmp_coef.plot(kind='barh')
plt.title("Feature importance using Lasso Model", size=20)
plt.xlabel("Coefficient")<jupyter_output><empty_output><jupyter_text>Predicting on new movies using the entire dataset<jupyter_code># Whole dataset training with lasso
X, y = model_df.drop('rating', axis=1), movie_df.rating

scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

lm_lasso.fit(X_scaled, y)
lasso_preds = lm_lasso.predict(X_scaled)

lasso_test_rmse = mean_squared_error(y, lasso_preds, squared=False)
lasso_test_mae = mean_absolute_error(y, lasso_preds)
lasso_test_r2 = lm_lasso.score(X_scaled, y)

print(lasso_test_r2)
print(lasso_test_rmse)
print(lasso_test_mae)
coef = pd.Series(lm_lasso.coef_ , index=XL.columns)
coef<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14281230382925297, 'fraction_numerical': 0.07432517263025738, 'mean_word_length': 5.148591277499035, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '288326', 'license': 'MIT', 'n_tokens_mistral': 7381, 'n_tokens_neox': 6700, 'n_words': 1069, 'path': 'workflow/3_regression IMDB.ipynb', 'repo_name': 'danielle-j-wang/regression_project', 'size': '16268'}