<jupyter_start><jupyter_text>Final Project Baseline: House Prices- Advanced Regression Techniques Author: <NAME>, <NAME>, <NAME> Date: Fall 2021 Course: w207 Machine Learning  Supplementary Notebook for PCA assessment, and its effect on Random Forest Regressor<jupyter_code># This tells matplotlib not to try opening a new window for each plot.
from pathlib import Path
import pandas as pd
import random
import warnings

import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator

# sklearn
import sklearn
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.decomposition import PCA

from sklearn.feature_selection import mutual_info_regression
from sklearn.ensemble import RandomForestRegressor



# internal modules
from configurations import args
from data_loader import DataLoader
from random_forest_regressor import RandomForestRegressor

%matplotlib inline


warnings.filterwarnings('ignore')
# download data file if it is not ready
data_file = Path(args.data_path)
if data_file.is_file():
    print("Datafile is already loaded.")
else:
    !curl -L "https://drive.google.com/uc?export=download&id=1ortEJfmlpt9-dbg5f6cTDt5nicswf5wT" > 'test.csv'
    !curl -L "https://drive.google.com/uc?export=download&id=1EG5jP5RDEIsNAMaF2m42KOyz-ZVjC8wS" > 'train.csv'
data_loader = DataLoader(args, post_eda=False)
df = data_loader.df

# lock the seed to have repeatable results
random.seed(100)
missing=df.isnull().sum().sort_values(ascending=False).head(20)
missing

#######New

missing = missing[missing > 0]
percent_missing = missing * 100 / df.shape[0]
pd.concat([missing, percent_missing], axis=1, 
          keys=['Missing Values', 'Percentage Missing']).\
          sort_values(by="Missing Values", ascending=False)
# Fill missing value of 'PoolQC','MiscFeature','Alley','Fence','FireplaceQu' as NonExist. Leave LotFrontage for further investigation
df[['PoolQC','MiscFeature','Alley','Fence','FireplaceQu']]=df[['PoolQC','MiscFeature','Alley','Fence','FireplaceQu']].fillna('NonExist')
missing_with_replace=df.isnull().sum().sort_values(ascending=False).head(20)
missing_with_replace

#######New

missing_with_replace = missing_with_replace[missing_with_replace > 0]
percent_missing = missing_with_replace * 100 / df.shape[0]
pd.concat([missing_with_replace, percent_missing], axis=1, 
          keys=['Missing Values After filling Non_Exist', 'Percentage Missing']).\
          sort_values(by="Missing Values After filling Non_Exist", ascending=False)<jupyter_output><empty_output><jupyter_text>Run PCA  Initial Look at the data:<jupyter_code># split data into Test and train
df_PCA_numeric_train, df_PCA_numeric_test, df_SalePrice_train, df_SalePrice_test = train_test_split(
    data_loader.df_X, data_loader.df_y, test_size=0.10, random_state=1)
df_PCA_numeric_train = df_PCA_numeric_train._get_numeric_data()
df_PCA_numeric_test = df_PCA_numeric_test._get_numeric_data()

missing=df_PCA_numeric_train.isnull().sum().sort_values(ascending=False).head(20)
missing

#######New

missing = missing[missing > 0]
percent_missing = missing * 100 / df.shape[0]
pd.concat([missing, percent_missing], axis=1, 
          keys=['Missing Values', 'Percentage Missing']).\
          sort_values(by="Missing Values", ascending=False)
missing=df_PCA_numeric_test.isnull().sum().sort_values(ascending=False).head(20)
missing

#######New

missing = missing[missing > 0]
percent_missing = missing * 100 / df.shape[0]
pd.concat([missing, percent_missing], axis=1, 
          keys=['Missing Values', 'Percentage Missing']).\
          sort_values(by="Missing Values", ascending=False)
# Fill missing value of 'PoolQC','MiscFeature','Alley','Fence','FireplaceQu' as NonExist. Leave LotFrontage for further investigation
df_PCA_numeric_train[['LotFrontage','GarageYrBlt','MasVnrArea']]=df_PCA_numeric_train[['LotFrontage','GarageYrBlt','MasVnrArea']].fillna('0')
missing_with_replace=df_PCA_numeric_train.isnull().sum().sort_values(ascending=False).head(20)
missing_with_replace

#######New

missing_with_replace = missing_with_replace[missing_with_replace > 0]
percent_missing = missing_with_replace * 100 / df.shape[0]
pd.concat([missing_with_replace, percent_missing], axis=1, 
          keys=['Missing Values After filling Non_Exist', 'Percentage Missing']).\
          sort_values(by="Missing Values After filling Non_Exist", ascending=False)
# Fill missing value of 'PoolQC','MiscFeature','Alley','Fence','FireplaceQu' as NonExist. Leave LotFrontage for further investigation
df_PCA_numeric_test[['LotFrontage','GarageYrBlt']]=df_PCA_numeric_test[['LotFrontage','GarageYrBlt']].fillna('0')
missing_with_replace=df_PCA_numeric_test.isnull().sum().sort_values(ascending=False).head(20)
missing_with_replace

#######New

missing_with_replace = missing_with_replace[missing_with_replace > 0]
percent_missing = missing_with_replace * 100 / df.shape[0]
pd.concat([missing_with_replace, percent_missing], axis=1, 
          keys=['Missing Values After filling Non_Exist', 'Percentage Missing']).\
          sort_values(by="Missing Values After filling Non_Exist", ascending=False)
def PCA_Variance():
    principal_components=PCA()
    principal_components.fit(df_PCA_numeric_train)
    
    K = range(30)
    for k in K:
        partial_variance_cumsum = principal_components.explained_variance_ratio_.cumsum()
        partial_variance = principal_components.explained_variance_ratio_
        
    for k in K:
        print ("k =", k, " Variance =", partial_variance[k])
 
    
    plt.figure(figsize=(12, 4))
    plt.plot(K, partial_variance_cumsum[K],'--bo')
    plt.title('Accumulated Fraction of the total variance in the training data explained by the first k principal components')
    plt.xlabel('k')
    plt.ylabel('Fraction of variance (accum.)')
    plt.axis()
    for i_x,i_y in zip(K,partial_variance_cumsum[K]):
        plt.text(i_x,i_y,'(PC{})'.format(i_x),fontsize = 12)
        
        
    plt.show() 
    
PCA_Variance()
def PCA_variances_2D():   

    PCA_2D = PCA(n_components=10)

    PCA_transformed=PCA_2D.fit_transform(df_PCA_numeric_train)

    fig, axes = plt.subplots(3, 2, figsize=(10, 10))
    fig.suptitle('X_train PCA Transformations')
    axes[0, 0].plot(PCA_transformed[:,0],PCA_transformed[:,1], 'bo',markersize=2)
    axes[0, 0].set_title('PCA0 vs PCA1')
    
    axes[0, 1].plot(PCA_transformed[:,1], PCA_transformed[:,2], 'bo',markersize=2)
    axes[0, 1].set_title('PCA1 vs PCA2')

    axes[1, 0].plot(PCA_transformed[:,2], PCA_transformed[:,3], 'bo',markersize=2)
    axes[0, 1].set_title('PCA2 vs PCA3')

    axes[1, 1].plot(PCA_transformed[:,3], PCA_transformed[:,4], 'bo',markersize=2)
    axes[1, 1].set_title('PCA3 vs PCA4')
    
    axes[2, 0].plot(PCA_transformed[:,4], PCA_transformed[:,5], 'bo',markersize=2)
    axes[2, 0].set_title('PCA4 vs PCA5')
    
    axes[2, 1].plot(PCA_transformed[:,5], PCA_transformed[:,6], 'bo',markersize=2)
    axes[2, 1].set_title('PCA5 vs PCA6')
    
    plt.show()


PCA_variances_2D()

def PCA_Final():   
 

    PCA_2D = PCA(n_components=6)

    PCA_transformed=PCA_2D.fit_transform(df_PCA_numeric_train)

    fig, axes = plt.subplots(3, 2, figsize=(10, 10))
    fig.suptitle('Sale Price Vs X_train PCA Transformations')
    axes[0, 0].plot(PCA_transformed[:,0],df_SalePrice_train, 'bo',markersize=2)
    axes[0, 0].set_title('PCA0')
    
    axes[0, 1].plot(PCA_transformed[:,1],df_SalePrice_train, 'bo',markersize=2)
    axes[0, 1].set_title('PCA1')


    axes[1, 0].plot(PCA_transformed[:,2],df_SalePrice_train, 'bo',markersize=2)
    axes[1, 0].set_title('PCA2')


    axes[1, 1].plot(PCA_transformed[:,3],df_SalePrice_train, 'bo',markersize=2)
    axes[1, 1].set_title('PCA3')
    
    axes[2, 0].plot(PCA_transformed[:,4],df_SalePrice_train, 'bo',markersize=2)
    axes[2, 0].set_title('PCA4')
    
    axes[2, 1].plot(PCA_transformed[:,5],df_SalePrice_train, 'bo',markersize=2)
    axes[2, 1].set_title('PCA5')
    
    for ax in axes.flat:
        ax.set(ylabel='Sale Price')
    
    plt.show()
    print('========================================================================================')
    print('')
    
    #print(PCA_transformed)
    
    df_X_processed = pd.DataFrame(PCA_transformed, columns = ['PCA0','PCA1','PCA2','PCA3','PCA4','PCA5'])
    #print(df_X_processed)
    
    #print(df_SalePrice_train)
    
    
    
    PCA_transformed_test=PCA_2D.fit_transform(df_PCA_numeric_test)

    fig, axes = plt.subplots(3, 2, figsize=(10, 10))
    fig.suptitle('Sale Price Vs X_test PCA Transformations')
    axes[0, 0].plot(PCA_transformed_test[:,0],df_SalePrice_test, 'bo',markersize=2)
    axes[0, 0].set_title('PCA0')
    
    axes[0, 1].plot(PCA_transformed_test[:,1],df_SalePrice_test, 'bo',markersize=2)
    axes[0, 1].set_title('PCA1')


    axes[1, 0].plot(PCA_transformed_test[:,2],df_SalePrice_test, 'bo',markersize=2)
    axes[1, 0].set_title('PCA2')


    axes[1, 1].plot(PCA_transformed_test[:,3],df_SalePrice_test, 'bo',markersize=2)
    axes[1, 1].set_title('PCA3')
    
    axes[2, 0].plot(PCA_transformed_test[:,4],df_SalePrice_test, 'bo',markersize=2)
    axes[2, 0].set_title('PCA4')
    
    axes[2, 1].plot(PCA_transformed_test[:,5],df_SalePrice_test, 'bo',markersize=2)
    axes[2, 1].set_title('PCA5')
    
    for ax in axes.flat:
        ax.set(ylabel='Sale Price')
    
    plt.show()
    print('========================================================================================')
    print('')
    
    #print(PCA_transformed_test)
    
    df_X_processed_test = pd.DataFrame(PCA_transformed_test, columns = ['PCA0','PCA1','PCA2','PCA3','PCA4','PCA5'])
    #print(df_X_processed_test)
    
    
    
    X_train, X_test, y_train, y_test = df_X_processed, df_X_processed_test,df_SalePrice_train,df_SalePrice_test
    regressor = RandomForestRegressor()
    regressor.fit(X_train, y_train)

    # plot dignostics
    fig, axes = plt.subplots(1, 1, figsize=(10, 10))
    plt.scatter(y_test, regressor.predict(X_test))
    plt.title('Random Forest Regressor with PCA')
    plt.xlabel('SalePrice')
    plt.ylabel('Predicted SalePrice')
    
    plt.show()
    print('========================================================================================')
    print('RMSE (Random Forest Regressor) = ',mean_squared_error(y_test, regressor.predict(X_test), squared=False))
    print('========================================================================================')



PCA_Final()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1702957117048588, 'fraction_numerical': 0.025482552058571837, 'mean_word_length': 5.397810218978102, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '575815', 'license': 'Unlicense', 'n_tokens_mistral': 3965, 'n_tokens_neox': 3665, 'n_words': 625, 'path': 'notebook_5_pca.ipynb', 'repo_name': 'pmotameni/mids-w207-final-project', 'size': '10539'}