<jupyter_start><jupyter_text>Import Necessary Packages<jupyter_code>import pandas as pd
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor
from sklearn.metrics import mean_squared_error as MSE
from sklearn.model_selection import train_test_split,GridSearchCV
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import LabelEncoder
from matplotlib import pyplot as plt
from matplotlib.pyplot import figure<jupyter_output><empty_output><jupyter_text>Defining some global variables<jupyter_code># Defining some global variables
SEED = 42
# Ingest Data
# Training dataset
df=pd.read_csv("E:\\Jupyter\\Kaggle\\House Prices Advanced Regression Techniques\\train.csv")
X = df.iloc[:,:-1]
# Target Variable - House sale price
y = df.loc[:,['SalePrice']]

# Testing dataset
testdata = pd.read_csv("E://Jupyter//Kaggle//House Prices Advanced Regression Techniques//test.csv")<jupyter_output><empty_output><jupyter_text>Data Exploration Firstly, having a look at columns<jupyter_code>X.columns<jupyter_output><empty_output><jupyter_text>Total number of columns and its distribution<jupyter_code>len(X.columns)
# Prepare a list for different type of columns:
list_null_cols = []
list_numeric_cols = []
list_categorical_cols = []

for col in X.columns:
    col_dtype = X[col].dtype
    if (df[col].isnull().sum()>0):
        list_null_cols.append(col)
    elif col_dtype=="O":
        list_categorical_cols.append(col)
    else:
        list_numeric_cols.append(col)
len(list_null_cols)
len(list_numeric_cols)
len(list_categorical_cols)
#Checking listing down visualizations
plt.figure(1)
plt.scatter(X.loc[:,'MSZoning'],y.loc[:,'SalePrice'])
plt.figure(2)
plt.scatter(X.loc[:,'MSSubClass'],y.loc[:,'SalePrice'])
plt.show()<jupyter_output><empty_output><jupyter_text>Looking at all features Visualization of all features<jupyter_code>i=0
for col in list_numeric_cols:
    plt.figure(i+1)
    plt.scatter(X.loc[:,col],y.loc[:,'SalePrice'])
    plt.title(col)
    i=i+1<jupyter_output>C:\Users\sscaria\AppData\Local\Continuum\anaconda3\lib\site-packages\matplotlib\pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).
  max_open_warning, RuntimeWarning)<jupyter_text>So we have below types of columns:1. With categorical data (27)2. With numeric data (35)3. With null values (19)  Dealing with categorical data (27)  Converting Categorical Data to Numerical data<jupyter_code>dfc = X.copy()
dict_categorical_col = {}
for col in list_categorical_cols:
    #Training data
    dict_categorical_col[col]=dfc[col].drop_duplicates().reset_index(drop=True).reset_index()
    dict_categorical_col[col].columns = ['label',col]
    len1=len(dict_categorical_col[col])
    df_temp = pd.DataFrame({'label':[len1],col:['unknown']},columns=['label',col],index=[len1])
    dict_categorical_col[col] = dict_categorical_col[col].append(df_temp)<jupyter_output><empty_output><jupyter_text>Cleaning Train & Test data<jupyter_code>testdata1 = testdata.copy()
X1 = X.copy()
for col in list_categorical_cols:
    #Mapping with dictionary for Train data
    X1 = pd.merge(X1,dict_categorical_col[col],on=col,how='left')
    X1 = X1.drop(columns=col)
    X1.rename(columns={'label':col},inplace=True)
    
    #Checking for null values in Test data
    testdata1[col] = testdata1[col].fillna("unknown")
    #Mapping with dictionary for Test data
    testdata1 = pd.merge(testdata1,dict_categorical_col[col],on=col,how='left')
    testdata1 = testdata1.drop(columns=col)
    testdata1.rename(columns={'label':col},inplace=True)
testdata1['Street'].unique()<jupyter_output><empty_output><jupyter_text>Dealing with numeric data (35) First deal with missing data related to numeric columns in test data<jupyter_code>imputer1 = SimpleImputer()
testdata2 = testdata1.copy()
testdata2[list_numeric_cols] = pd.DataFrame(imputer1.fit_transform(testdata1[list_numeric_cols]))
testdata2.columns = testdata1.columns<jupyter_output><empty_output><jupyter_text>Calculation Correlation<jupyter_code>from scipy.stats import pearsonr
i_index=0
pearson_corr_df = pd.DataFrame(columns=['column','correlation'],index=[-1])
for col in list_numeric_cols:
    corr,_ = pearsonr(X1.loc[:,col],y.loc[:,'SalePrice'])
    dftest = pd.DataFrame({'column': [col],'correlation':[corr]},columns=['column','correlation'],index=[i_index])
    pearson_corr_df = pearson_corr_df.append(dftest)
    i_index=i_index+1
pearson_corr_df = pearson_corr_df.dropna()
pearson_corr_df.loc[:,'correlation'] = pearson_corr_df.loc[:,'correlation'].abs()
pearson_corr_df=pearson_corr_df.sort_values(by='correlation')
pearson_corr_df.loc[(pearson_corr_df['correlation']>0.5),['correlation']]
plt.figure(figsize=(5,10))
threshold = 0.5
plot_df = pearson_corr_df.loc[(pearson_corr_df['correlation']>threshold),:]
plt.barh(plot_df.loc[:,'column'],plot_df.loc[:,'correlation'])
plt.show()
type(pearson_corr_df.loc[(pearson_corr_df['correlation']>threshold),['column']])
pearson_corr_df.loc[(pearson_corr_df['correlation']>threshold),['correlation']]<jupyter_output><empty_output><jupyter_text>Dealing with Category 3: With null values (19)<jupyter_code>list_null_cols
X.groupby('Fence').size()
var_alley = X.loc[:,['Alley']].fillna("No Alley")
plt.scatter(var_alley.loc[:,'Alley'],y.loc[:,'SalePrice'])<jupyter_output><empty_output><jupyter_text>Model Building<jupyter_code>#Step 2: Building initial model on all numerical columns excluding 
#feature_list = list_numeric_cols
feature_list = list_numeric_cols + list_categorical_cols
X1_features = X1[feature_list]
X1_features.shape
y.shape
#Step 3: Define Train & Test Sets
X_train,X_test,y_train,y_test = train_test_split(X1_features, y, random_state=SEED, test_size=0.2)
y_train.shape
#Step 4: Define a Model
dt = DecisionTreeRegressor(random_state=SEED, max_depth=10)
#Step 5: Fitting the model

dt.fit(X_train,y_train)<jupyter_output><empty_output><jupyter_text>Calculating Training Error<jupyter_code>y_train_pred = dt.predict(X_train)
model_train_error = MSE(y_train,y_train_pred)**(1/2)<jupyter_output><empty_output><jupyter_text>Calculation Testing Error<jupyter_code>y_test_pred = dt.predict(X_test)
model_test_error = MSE(y_test,y_test_pred)**(1/2)
model_train_error
model_test_error<jupyter_output><empty_output><jupyter_text>Grid Search<jupyter_code>params_dt = {'max_depth': range(4,11),
            'min_samples_leaf':[0.04,0.06,0.08],
            'max_features':[0.2,0.3,0.4,0.5,0.6,0.7,0.8]}
grid_dt = GridSearchCV(estimator = dt, 
                       param_grid = params_dt, 
                       scoring ='neg_mean_squared_error',
                       cv=10,
                       n_jobs=-1
                      )
grid_dt.fit(X1_features,y)
grid_dt.best_params_
selectedmodel = grid_dt.best_estimator_
#Calculating Training error
y_train_pred_selectedmodel= selectedmodel.predict(X_train)
y_train_pred_selectedmodel_trainerror = MSE(y_train,y_train_pred_selectedmodel)**(1/2)
#Calculating Testing error
y_test_pred_selectedmodel= selectedmodel.predict(X_test)
y_test_pred_selectedmodel_testerror = MSE(y_test,y_test_pred_selectedmodel)**(1/2)
y_train_pred_selectedmodel_trainerror
y_test_pred_selectedmodel_testerror<jupyter_output><empty_output><jupyter_text>Now lets repeat the same for Gradient Boosting<jupyter_code>gbt = GradientBoostingRegressor(random_state=SEED)
# params_gbt = { 'n_estimators':[100,150,200,250,300,350,400,450,500],
#               'max_depth': [5,6,7,10,50,80,100]
#              }
# Grid Search gave best depth as 6 & number of estimators as 250
params_gbt = { 'n_estimators':[250],
              'max_depth': [6]
             }
gbt_gcv = GridSearchCV(estimator = gbt,
                       param_grid = params_gbt,
                       scoring='neg_mean_squared_error',
                       cv=10,
                       n_jobs=-1
                      )
gbt_gcv.fit(X1_features,y)
gbt_gcv.best_params_
selectedmodel_gbt = gbt_gcv.best_estimator_
#Calculating train error
y_train_pred_selectedmodel_gbt= selectedmodel_gbt.predict(X_train)
y_train_pred_selectedmodel_trainerror_gbt = MSE(y_train,y_train_pred_selectedmodel_gbt)**(1/2)
#Calculating test error
y_test_pred_selectedmodel_gbt= selectedmodel_gbt.predict(X_test)
y_test_pred_selectedmodel_testerror_gbt = MSE(y_test,y_test_pred_selectedmodel_gbt)**(1/2)
y_train_pred_selectedmodel_trainerror_gbt
y_test_pred_selectedmodel_testerror_gbt<jupyter_output><empty_output><jupyter_text>Now lets look at using plotting trainerror & testerror for different max_depth<jupyter_code>import numpy as np
list_max_depth = range(1,8)
train_errors = list()
test_errors = list()
for treedepth in list_max_depth:
    dt_model = DecisionTreeRegressor(random_state=SEED, max_depth = treedepth)
    dt_model.fit(X_train,y_train)
    train_errors.append(dt_model.score(X_train,y_train))
    test_errors.append(dt_model.score(X_test,y_test))
plt.plot(np.array(list_max_depth),np.array(train_errors), label='Train')
plt.plot(np.array(list_max_depth),np.array(test_errors), label='Test')
plt.legend(loc='upper left')
plt.show()<jupyter_output><empty_output><jupyter_text>For Predictions<jupyter_code>y_predictions = selectedmodel_gbt.predict(testdata2[feature_list])
output = pd.DataFrame({'Id': testdata.Id,
                       'SalePrice': y_predictions})
output.to_csv('submission1.csv', index=False)
selectedmodel_gbt.feature_importances_
importances_dt = pd.Series(selectedmodel_gbt.feature_importances_,index=testdata2[feature_list].columns)
sorted_importances_dt = importances_dt.sort_values()
plt.figure(figsize=(5,10))
sorted_importances_dt.plot(kind='barh',color='lightgreen')

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12016560638190447, 'fraction_numerical': 0.01827728971018883, 'mean_word_length': 6.548780487804878, 'pattern_counts': {'":': 1, '<': 68, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '504721', 'license': 'Apache-2.0', 'n_tokens_mistral': 3702, 'n_tokens_neox': 3516, 'n_words': 535, 'path': 'housingprice/notebooks/DecisionTree.ipynb', 'repo_name': 'nirvana1707/helloworld', 'size': '9903'}