<jupyter_start><jupyter_text>Sampling<jupyter_code>from sklearn.model_selection import train_test_split

traindf, testdf = train_test_split(rawdf, train_size = 0.8, random_state = 143)<jupyter_output><empty_output><jupyter_text>"Source" Column In Train & Test<jupyter_code>traindf["Source"]   = "Train"
testdf["Source"]    = "Test"
predictdf["Source"] = "Predict"<jupyter_output><empty_output><jupyter_text>Adding "SalePrice" Column in predictdf<jupyter_code>predictdf["SalePrice"] = 0;
traindf.shape
testdf.shape
predictdf.shape<jupyter_output><empty_output><jupyter_text>Concat Train, Test and Predict Dataset<jupyter_code>furaw = pd.concat([traindf, testdf, predictdf], axis = 0);
furaw.info()
furaw.isnull().sum()<jupyter_output><empty_output><jupyter_text>Drop "Id"  and "Month" Column<jupyter_code>furaw = furaw.drop(["Id"], axis = 1)

furaw = furaw.drop(["MoSold"], axis = 1)<jupyter_output><empty_output><jupyter_text>Data Pre_Processing "Removing columns that possess no credible value to model buidling and those those with more than 70% of missing values"<jupyter_code>furaw = furaw.drop(["Alley","MiscFeature","GarageYrBlt",
                    "GarageFinish","LotFrontage"], axis = 1);
furaw.shape<jupyter_output><empty_output><jupyter_text>Columns with Year Data<jupyter_code>furaw["Build_Age"] = furaw["YrSold"] - furaw["YearBuilt"];furaw.shape<jupyter_output><empty_output><jupyter_text>Drop Year Sold, YearBuilt & Remodel Year<jupyter_code>furaw = furaw.drop(["YrSold", "YearBuilt", "YearRemodAdd"], axis = 1); furaw.shape<jupyter_output><empty_output><jupyter_text>Columns with object variable<jupyter_code>furaw["MSSubClass"]    = furaw["MSSubClass"].astype(object)

furaw["OverallQual"]  = furaw["OverallQual"].astype(object)

furaw["OverallCond"]   = furaw["OverallCond"].astype(object)<jupyter_output><empty_output><jupyter_text>Missing Value Imputation  Category columns replacing NA's with Not present NoB - No Basement<jupyter_code>furaw['BsmtQual'].replace(np.nan, "NoB", inplace = True)

furaw['BsmtExposure'].replace(np.nan, "NoB", inplace = True)

furaw['BsmtFinType1'].replace(np.nan, "NoB", inplace = True)

furaw['BsmtFinType2'].replace(np.nan, "NoB", inplace = True)

furaw['BsmtCond'].replace(np.nan, "NoB", inplace = True)<jupyter_output><empty_output><jupyter_text>NoFi -No Fire<jupyter_code>furaw['FireplaceQu'].replace(np.nan, "NoFi", inplace = True)<jupyter_output><empty_output><jupyter_text>NoG - No Garage<jupyter_code>furaw['GarageType'].replace(np.nan, "NoG", inplace = True)

furaw['GarageQual'].replace(np.nan, "NoG", inplace = True)

furaw['GarageCond'].replace(np.nan, "NoG", inplace = True)<jupyter_output><empty_output><jupyter_text>NoP - No Pool<jupyter_code>furaw['PoolQC'].replace(np.nan, "NoP", inplace = True)<jupyter_output><empty_output><jupyter_text>0 - Number of Bathrooms<jupyter_code>furaw['BsmtFullBath'].replace(np.nan, 0, inplace = True)

furaw['BsmtHalfBath'].replace(np.nan, 0, inplace = True)<jupyter_output><empty_output><jupyter_text>NoFe - No Fence<jupyter_code>furaw['Fence'].replace(np.nan, "NoFe", inplace = True)<jupyter_output><empty_output><jupyter_text>Replaced by Mode value<jupyter_code>furaw['KitchenQual'].replace(np.nan, "TA", inplace = True)

furaw['SaleType'].replace(np.nan, "WD", inplace = True)

furaw['Functional'].replace(np.nan, "TF", inplace = True)

furaw['MasVnrType'].replace(np.nan, "None", inplace = True)

furaw['Electrical'].replace(np.nan, "SBrKr", inplace = True)

furaw['Utilities'].replace(np.nan, "AllPub", inplace = True)

furaw['Exterior1st'].replace(np.nan, "VinylSd", inplace = True)

furaw['Exterior2nd'].replace(np.nan, "VinylSd", inplace = True)

furaw['MSZoning'].replace(np.nan, "RLD", inplace = True)<jupyter_output><empty_output><jupyter_text>Replaced by Median value<jupyter_code>furaw['GarageCars'].replace(np.nan, 2, inplace = True)

furaw['MasVnrArea'].replace(np.nan, 0, inplace = True)

furaw['GarageArea'].replace(np.nan, 477, inplace = True)

furaw['BsmtFinSF2'].replace(np.nan, 0, inplace = True)

furaw['BsmtUnfSF'].replace(np.nan, 467, inplace = True)

furaw['TotalBsmtSF'].replace(np.nan, 989, inplace = True)

furaw['BsmtFinSF1'].replace(np.nan, 368, inplace = True)
furaw.isnull().sum()
furaw.shape<jupyter_output><empty_output><jupyter_text>Corrected Dataset Extracted!<jupyter_code>furaw.to_csv("Fullraw.csv", index = False)<jupyter_output><empty_output><jupyter_text>Correlation Check<jupyter_code>import matplotlib.pyplot as plt
plt.figure(figsize=(20,20))

corrDf = furaw[furaw["Source"] == "Train"].corr()

ax = sns.heatmap( corrDf, 
                  vmin=-1, vmax=1, center=0,
                  cmap=sns.diverging_palette(20, 220, n=200),
                  square=True )
ax.set_xticklabels(ax.get_xticklabels(),
                   rotation=45,
                   horizontalalignment='right');<jupyter_output><empty_output><jupyter_text>Dummy Variable Creation<jupyter_code>fullRaw = pd.get_dummies(furaw, drop_first = True) 

fullRaw.shape<jupyter_output><empty_output><jupyter_text>Sampling<jupyter_code>trainDf      = fullRaw[fullRaw['Source_Train'] == 1].drop(['Source_Train', 'Source_Test'], axis = 1).copy()

testDf       = fullRaw[fullRaw['Source_Test'] == 1].drop(['Source_Train', 'Source_Test'], axis = 1).copy()

predictDf    = fullRaw[(fullRaw['Source_Train'] == 0) & 
                        (fullRaw['Source_Test'] == 0)].drop(['Source_Train', 'Source_Test'], axis = 1).copy()<jupyter_output><empty_output><jupyter_text>Divide into Xs (Indepenedents) and Y (Dependent)<jupyter_code>trainX = trainDf.drop(['SalePrice'], axis = 1).copy()
trainY = trainDf['SalePrice'].copy()

testX  = testDf.drop(['SalePrice'], axis = 1).copy()
testY  = testDf['SalePrice'].copy()<jupyter_output><empty_output><jupyter_text>Add intercept Column<jupyter_code>from statsmodels.api import add_constant

trainX    = add_constant(trainX)
predictDf = add_constant(predictDf)
testX     = add_constant(testX, has_constant='add')
print( "trainX: " , trainX.shape)

print( "testX: "  , testX.shape)

print( "predictDf: "  , predictDf.shape)<jupyter_output>trainX:  (1168, 278)
testX:  (292, 278)
predictDf:  (1459, 279)<jupyter_text>VIF Check<jupyter_code>from statsmodels.stats.outliers_influence import variance_inflation_factor

tempMaxVIF = 5 
maxVIF     = 5
trainXCopy = trainX.copy()
counter    = 1
highVIFColumnNames = []

while (tempMaxVIF >= maxVIF):
    
    print(counter)
    
    tempVIFDf = pd.DataFrame()
    
    tempVIFDf['VIF'] = [variance_inflation_factor(trainXCopy.values, i) for i in range(trainXCopy.shape[1])]
    
    tempVIFDf['Column_Name'] = trainXCopy.columns
    
    tempVIFDf.dropna(inplace=True)
    
    tempColumnName = tempVIFDf.sort_values(["VIF"], ascending = False).iloc[0,1]
    
    tempMaxVIF = tempVIFDf.sort_values(["VIF"], ascending = False).iloc[0,0]
    
    print(tempColumnName)
    
    if (tempMaxVIF >= maxVIF):
        
        trainXCopy = trainXCopy.drop(tempColumnName, axis = 1)    
        highVIFColumnNames.append(tempColumnName)
    
    counter = counter + 1

highVIFColumnNames<jupyter_output>1
OverallCond_6
2
OverallQual_4
3
BsmtQual_NoB
4
BldgType_Duplex
5
BsmtCond_NoB
6
GrLivArea
7
BsmtFinType2_NoB
8
GarageType_NoG
9
GarageCond_NoG
10
Electrical_Mix
11
Exterior2nd_CBlock
12
BsmtFinSF1
13
const
14
PoolQC_NoP
15
GarageCond_TA
16
RoofMatl_CompShg
17
Condition2_Norm
18
Heating_GasA
19
GarageQual_TA
20
RoofStyle_Gable
21
ExterCond_TA
22
Street_Pave
23
MSZoning_RL
24
GarageType_Attchd
25
Exterior1st_VinylSd
26
1stFlrSF
27
KitchenAbvGr
28
BsmtFinType2_Unf
29
Functional_Typ
30
HouseStyle_1Story
31
TotRmsAbvGrd
32
MasVnrType_None
33
Exterior2nd_MetalSd
34
BldgType_TwnhsE
35
GarageCars
36
ExterQual_TA
37
TotalBsmtSF
38
BsmtCond_TA
39
HouseStyle_2Story
40
FireplaceQu_NoFi
41
SaleType_WD
42
CentralAir_Y
43
BldgType_2fmCon
44
SaleCondition_Partial
45
BedroomAbvGr
46
Build_Age
47
Condition1_Norm
48
FullBath
49
BsmtFinType1_NoB
50
LandContour_Lvl
51
PavedDrive_Y
52
Exterior2nd_VinylSd
53
Fence_NoFe
54
Exterior1st_CemntBd
55
HouseStyle_SLvl
56
Electrical_SBrkr
57
KitchenQual_TA
58
Garag[...]<jupyter_text>Exclude 'const' column from getting dropped/ removed<jupyter_code>highVIFColumnNames.remove('const') 

highVIFColumnNames
trainX    = trainX.drop(highVIFColumnNames, axis = 1)

testX     = testX.drop(highVIFColumnNames, axis = 1)

predictDf = predictDf.drop(highVIFColumnNames, axis = 1)<jupyter_output><empty_output><jupyter_text>Model Building<jupyter_code>from statsmodels.api import OLS

m1ModelDef = OLS(trainY, trainX)

m1ModelBuild = m1ModelDef.fit() 

m1ModelBuild.summary()<jupyter_output><empty_output><jupyter_text>Extract/ Identify p-values from model<jupyter_code>dir(m1ModelBuild)

m1ModelBuild.pvalues<jupyter_output><empty_output><jupyter_text>Model Optimization<jupyter_code>tempMaxPValue = 0.1
maxPValue = 0.1
trainXCopy = trainX.copy()
counter = 1
highPValueColumnNames = []

while (tempMaxPValue >= maxPValue):
    
    print(counter)    
    
    tempModelDf = pd.DataFrame()    
    Model = OLS(trainY, trainXCopy).fit()
    tempModelDf['PValue'] = Model.pvalues
    tempModelDf['Column_Name'] = trainXCopy.columns
    tempModelDf.dropna(inplace=True) 
    tempColumnName = tempModelDf.sort_values(["PValue"], ascending = False).iloc[0,1]
    tempMaxPValue = tempModelDf.sort_values(["PValue"], ascending = False).iloc[0,0]
    
    if (tempMaxPValue >= maxPValue): 
        print(tempColumnName, tempMaxPValue)    
        trainXCopy = trainXCopy.drop(tempColumnName, axis = 1)    
        highPValueColumnNames.append(tempColumnName)
    
    counter = counter + 1

highPValueColumnNames<jupyter_output>1
BldgType_Twnhs 0.9988977622659606
2
SaleType_ConLw 0.9889299155231301
3
Exterior2nd_Other 0.9959824621136335
4
ExterQual_Fa 0.9846633023725417
5
Exterior2nd_Brk Cmn 0.9843017764252779
6
BsmtFinType1_LwQ 0.9841047763905257
7
Neighborhood_Sawyer 0.9770777794643816
8
BsmtFinType2_LwQ 0.9527949912100815
9
MiscVal 0.9306173539550545
10
MasVnrType_BrkFace 0.9287191302273518
11
MSSubClass_50 0.9294242001544506
12
Functional_Min1 0.9177385426482876
13
Foundation_Slab 0.9156817934156658
14
MSSubClass_150 0.9812993189254358
15
BsmtFinType1_Rec 0.9127268600617805
16
Exterior1st_AsphShn 0.9066708180183369
17
RoofStyle_Gambrel 0.9073359486308756
18
LotShape_IR3 0.9044132192119961
19
Electrical_FuseP 0.906225897178365
20
Exterior1st_BrkComm 0.8863158225139507
21
GarageCond_Fa 0.8978441075721533
22
Neighborhood_BrkSide 0.883225082589982
23
BsmtQual_Gd 0.8824954592180606
24
SaleType_Con 0.8816174986676852
25
Condition2_RRAe 0.9530521891108966
26
Utilities_NoSeWa 0.9253926281687022
27
MSZoning_RLD 0.[...]<jupyter_text>Final Model<jupyter_code>trainX = trainX.drop(highPValueColumnNames, axis = 1)

testX = testX.drop(highPValueColumnNames, axis = 1)

predictDf = predictDf.drop(highPValueColumnNames, axis = 1)
print("trainX: ", trainX.shape)

print("testX: ", testX.shape)

print("predictDf: ", predictDf.shape)
Model = OLS(trainY, trainX).fit()

Model.summary()<jupyter_output><empty_output><jupyter_text>Model Prediction<jupyter_code>Test_Pred = Model.predict(testX); Test_Pred[0:6]
testY[:6]<jupyter_output><empty_output><jupyter_text>Model Diagnostics<jupyter_code>plt.figure(figsize=(20,10))

sns.scatterplot(Model.fittedvalues, Model.resid)<jupyter_output><empty_output><jupyter_text>Normality of Errors:Check<jupyter_code>plt.figure(figsize=(20,10))

sns.distplot(Model.resid)<jupyter_output><empty_output><jupyter_text>MAPE<jupyter_code>np.mean(np.abs(((testY - Test_Pred)/testY)))*100<jupyter_output><empty_output><jupyter_text>RMSE<jupyter_code>np.sqrt(np.mean((testY - Test_Pred)**2))<jupyter_output><empty_output><jupyter_text>Prediction on Sample<jupyter_code>predictDf["Predicted_SalePrice"] = Model.predict(predictDf.drop(["SalePrice"], axis = 1))
predictDf.to_csv("Predicted_SalePrice.csv", index = True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12531581606872158, 'fraction_numerical': 0.06375273707259559, 'mean_word_length': 6.7010376134889755, 'pattern_counts': {'":': 0, '<': 138, '<?xml version=': 0, '>': 142, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '35', 'dir': 'jupyter-structured-clean-dedup', 'id': '159251', 'license': 'Apache-2.0', 'n_tokens_mistral': 5267, 'n_tokens_neox': 4728, 'n_words': 703, 'path': 'House_Price_01.ipynb', 'repo_name': 'Gautham-Py/Kaggle-House_Prices_Advanced_Regression_Technique', 'size': '11874'}