<jupyter_start><jupyter_text>Imports<jupyter_code>import pandas as pd
import numpy as np
import seaborn as sns
sns.set()
import matplotlib.pyplot as plt
%matplotlib inline

from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV

from sklearn.linear_model import ElasticNet, Ridge, Lasso
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, AdaBoostRegressor
from sklearn.tree import DecisionTreeRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from mlxtend.regressor import StackingCVRegressor

import warnings
warnings.filterwarnings("ignore")<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>train = pd.read_csv('input/train.csv', index_col='Id')
test = pd.read_csv('input/test.csv', index_col='Id')

# Remove rows with missing target, separate target from predictors
train.dropna(axis=0, subset=['SalePrice'], inplace=True)
y = train.SalePrice
train.drop(['SalePrice'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis<jupyter_code>train.head()
train.info()
train.describe()<jupyter_output><empty_output><jupyter_text>Data Visualization<jupyter_code>sns.distplot(y)
numerical_cols = train.select_dtypes(exclude='object').copy()

fig = plt.figure(figsize=(10,16))
for i in range(len(numerical_cols.columns)):
    fig.add_subplot(9,4,i+1)
    sns.distplot(numerical_cols.iloc[:,i].dropna())
    plt.xlabel(numerical_cols.columns[i])

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Play with outliers<jupyter_code>train = train.drop(train[train['LotFrontage'] > 200].index)
train = train.drop(train[train['LotArea'] > 100000].index)
train = train.drop(train[train['BsmtFinSF1'] > 4000].index)
train = train.drop(train[train['MasVnrArea'] > 1000].index)
train = train.drop(train[train['TotalBsmtSF'] > 5000].index)
train = train.drop(train[train['GrLivArea'] > 4000].index)
train = train.drop(train[train['PoolArea'] > 300].index)
train = train.drop(train[train['OpenPorchSF'] > 300].index)
figure = plt.figure(figsize=(12,20))

for i in range(len(numerical_cols.columns)):
    figure.add_subplot(9, 4, i+1)
    sns.scatterplot(numerical_cols.iloc[:,i], y)
    
plt.tight_layout()
plt.show()
def visualize_counts(feature):
    fig = plt.figure(figsize=(14,4))
    sns.countplot(x=feature, data=train)
    plt.xticks(rotation=90)
    plt.ylabel('Counts')
    plt.show()
visualize_counts('Neighborhood')
correlation = pd.concat([train, y], axis=1).corr()
f, ax = plt.subplots(figsize=(14,12))
plt.title('Correlation of numerical features and price', size=14)
sns.heatmap(correlation, mask=correlation < 0.1, linewidth=0.4, cmap='GnBu')
plt.show()
pd.concat([train, y], axis=1).corr()['SalePrice'].sort_values(ascending=False).head(10)<jupyter_output><empty_output><jupyter_text>Missing Values Analysis<jupyter_code>plt.figure(figsize=(30,10))
plt.title('Number of missing rows')
missing_count = pd.DataFrame(train.isnull().sum(), columns=['sum']).sort_values(by=['sum'],ascending=False).head(20).reset_index()
missing_count.columns = ['features','sum']
sns.barplot(x='features',y='sum', data = missing_count)
missing_values_count = train.isnull().sum().sort_values(ascending=False)
missing_values_count[missing_values_count > 0]
total_cells = np.product(train.shape)
total_missing = missing_values_count.sum()
percent_missing = total_missing / total_cells * 100
print(percent_missing)
missing_training_values = train.isnull().sum()
missing_training_values[missing_training_values > 0]
total_cells = np.product(test.shape)
total_missing = missing_values_count.sum()
percent_missing = total_missing / total_cells * 100
print(percent_missing)<jupyter_output>5.944768829005474<jupyter_text>Dealing With Missing Values<jupyter_code>train_test_data = [train, test]
def replace_nan(features):
    for feature in features:
        dataset[feature][dataset[feature].isnull()] = 'None'
nan_indicates_none = ['Alley', 'MasVnrType', 'BsmtQual','BsmtCond', 'BsmtFinType1', 
                      'BsmtFinType2', 'FireplaceQu', 'GarageType', 'GarageFinish', 
                      'GarageQual', 'GarageCond', 'BsmtExposure', 'PoolQC', 'Fence', 
                      'MiscFeature', 'Exterior1st', 'Exterior2nd']

for dataset in train_test_data:
    replace_nan(nan_indicates_none)
#LotFrontage
for dataset in train_test_data:
    dataset['LotFrontage'] = dataset['LotFrontage'].fillna(dataset['LotFrontage'].dropna().mean())
#Electrical
for dataset in train_test_data:
    dataset['Electrical'] = dataset['Electrical'].fillna(dataset['Electrical'].dropna().sort_values().index[0])
#GarageYrBlt, GarageArea, GarageCars
for dataset in train_test_data:
    dataset['GarageYrBlt'] = dataset['GarageYrBlt'].fillna(0)
    dataset['GarageArea'] = dataset['GarageArea'].fillna(0)
    dataset['GarageCars'] = dataset['GarageCars'].fillna(0)
#MasVnrArea
for dataset in train_test_data:
    dataset['MasVnrArea'] = dataset['MasVnrArea'].fillna(dataset['MasVnrArea'].dropna().mean())
#KitchenQual
for dataset in train_test_data:
    dataset['KitchenQual'] = dataset['KitchenQual'].fillna(dataset['KitchenQual'].dropna().sort_values().index[0])
#SaleType
for dataset in train_test_data:
    dataset['SaleType'] = dataset['SaleType'].fillna(dataset['SaleType'].dropna().sort_values().index[0])
#TotalBsmtSF
for dataset in train_test_data:
    dataset['TotalBsmtSF'] = dataset['TotalBsmtSF'].fillna(0)
#TotalBsmtSF
for dataset in train_test_data:
    dataset['BsmtFinSF1'] = dataset['BsmtFinSF1'].fillna(0)
    dataset['BsmtFinSF2'] = dataset['BsmtFinSF2'].fillna(0)
#BsmtFullBath, Functional, BsmtHalfBath, BsmtUnfSF
for dataset in train_test_data:
    dataset['BsmtFullBath'] = dataset['BsmtFullBath'].fillna(0)
    dataset['Functional'] = dataset['Functional'].fillna(dataset['Functional'].dropna().sort_values().index[0])
    dataset['BsmtHalfBath'] = dataset['BsmtHalfBath'].fillna(0)
    dataset['BsmtUnfSF'] = dataset['BsmtUnfSF'].fillna(0)
#MSZoning 
for dataset in train_test_data:
    dataset['MSZoning'] = dataset['MSZoning'].fillna(dataset['MSZoning'].dropna().sort_values().index[0])
#Utilities
for dataset in train_test_data:
    dataset['Utilities'] = dataset['Utilities'].fillna(dataset['Utilities'].dropna().sort_values().index[0])
missing = test.isnull().sum().sort_values(ascending=False)
missing[missing > 0]<jupyter_output><empty_output><jupyter_text>Feature Engineering (Can come back and address outliers)<jupyter_code>#log target to reduce skew
y_copy = y.copy()
y_copy = y_copy[train.index]
y_log = np.log(y_copy)
sns.distplot(y_log)
log_correlation = pd.concat([train, y_log], axis=1).corr()
f, ax = plt.subplots(figsize=(14,12))
plt.title('Correlation of numerical features and price', size=14)
sns.heatmap(log_correlation, mask=correlation<0.8, linewidth=0.4, cmap="GnBu")
plt.show()
#Remove highly correlated features which may reduce performance and one-hot encode
high_corr_attributes = ['GarageArea', 'GarageYrBlt', 'TotRmsAbvGrd',
                       'MiscVal', 'MSSubClass', 'MoSold', 'YrSold']

X = train.drop(high_corr_attributes, axis=1)
X = pd.get_dummies(X)


test_X = test.drop(high_corr_attributes, axis=1)
test_X = pd.get_dummies(test_X)

train_X, val_X, train_y, val_y = train_test_split(X, y_log, random_state=1)

#impute any missing values
imputer = SimpleImputer()
train_X = imputer.fit_transform(train_X)
val_X = imputer.transform(val_X)
#Invert log transformation
def inv_y(transformed_y):
    return np.exp(transformed_y)
#Quick train function
def score_model(model):
    model.fit(train_X, train_y)
    preds = model.predict(val_X)
    score = mean_absolute_error(inv_y(preds), inv_y(val_y))
    return score
#Save results to a series
results = pd.Series()
results.index.name = 'Model'
#XGBRegressor
xgb = XGBRegressor(learning_rate=0.014,
                   n_estimators=4500,
                   max_depth=4,
                   min_child_weight=0,
                   subsample=0.8,
                   colsample_bytree=0.4,
                   nthread=-1,
                   scale_pos_weight=2,
                   seed=42,)
results['XGB'] = score_model(xgb)
#KNeighbors
knn = KNeighborsRegressor(n_neighbors=3)
results['KNeighbors'] = score_model(knn)
#SVR
svr = SVR(C=1.0, epsilon=0.3)
results['SVR'] = score_model(svr)
#Random Forest
forest = RandomForestRegressor(random_state=1)
results['RandomForestRegressor'] = score_model(forest)
#AdaBoost
ada = AdaBoostRegressor(n_estimators=300, learning_rate=0.03, random_state=1)
results['ADA'] = score_model(ada)
#Gradient Boosting
grad = GradientBoostingRegressor(n_estimators=300, 
                                 learning_rate=0.03, 
                                 max_depth=4, 
                                 random_state=1)

results['Grad'] = score_model(grad)
#LightGBM
light = LGBMRegressor(n_estimators=3500,
                      num_leaves=5,
                      learning_rate=0.007,
                      max_bin=160,
                      bagging_fraction=0.35,
                      n_jobs=-1,
                      bagging_seed=40,
                      feature_fraction_seed=40,
                      bagging_freq=7,
                      feature_fraction=0.13,
                      min_data_in_leaf=8)

results['LightGBM'] = score_model(light)
#Lasso
lasso = Lasso(alpha=0.0005, random_state=1)
results['Lasso'] = score_model(lasso)
#ElasticNet
elastic = ElasticNet(alpha=0.02, random_state=1, l1_ratio=0.7)
results['ElasticNet'] = score_model(elastic)
#Ridge
ridge = Ridge(alpha=0.002, random_state=1)
results['Ridge'] = score_model(ridge)
results.sort_values()
#impute any missing values
imputer = SimpleImputer()
imputed_X = imputer.fit_transform(X)
n_folds = 10

lasso_scores = cross_val_score(lasso, imputed_X, y_log, scoring='neg_mean_squared_error', cv=n_folds)
lasso_rmse_cv_scores = np.sqrt(-lasso_scores)


print('Lasso Mean RMSE = ' + str(lasso_rmse_cv_scores.mean().round(decimals=3)))
print('Error std deviations = ' +str(lasso_rmse_cv_scores.std().round(decimals=3)))
lgbm_scores = cross_val_score(light, imputed_X, y_log, scoring='neg_mean_squared_error', cv=n_folds)
lgbm_rmse_cv_scores = np.sqrt(-lgbm_scores)


print('LGBM Mean RMSE = ' + str(lgbm_rmse_cv_scores.mean().round(decimals=3)))
print('Error std deviations = ' +str(lgbm_rmse_cv_scores.std().round(decimals=3)))
# Tuning Lasso
param_grid = [{'alpha': [0.0007, 0.0003, 0.0004, 0.0005, 0.0055, 0.005, 0.002]}]
best = Lasso()

grid = GridSearchCV(best, param_grid, cv=5, scoring='neg_mean_squared_error')
grid.fit(imputed_X, y_log)
grid.best_params_
# #Tuning LGBM
# second_best = LGBMRegressor()
# param_grid = {
#     'n_estimators' : [3500, 3000],
#     'num_leaves' : [5, 6],
#     'learning_rate' : [0.007],
#     'max_bin' : [100, 160, 180],
#     'bagging_fraction' : [0.35711, 0.66],
#     'bagging_seed' : [42, 88],
#     'feature_fraction_seed' : [42, 29],
#     'bagging_freq' : [7, 10],
#     'feature_fraction' : [0.13, 0.19],
#     'min_data_in_leaf' : [4, 8],
# }
# grid = GridSearchCV(second_best, param_grid, cv=5, scoring='neg_mean_squared_error')
# grid.fit(imputed_X, y_log)
# grid.best_params_
#ensure same alignment
final_train, final_test = X.align(test_X, join='left', axis=1)
final_test_imputed = imputer.transform(final_test)
#initialize final model and compute predictions
final_model = Lasso(alpha=0.0005, random_state=1)
final_train_imputed = imputer.fit_transform(final_train)
final_model.fit(final_train_imputed, y_log)

test_predictions = final_model.predict(final_test_imputed)
submission = pd.DataFrame({
    'Id' : test.index,
    'SalePrice' : inv_y(test_predictions)
})

submission.to_csv('submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1375811013142572, 'fraction_numerical': 0.026285143902844785, 'mean_word_length': 5.388416578108395, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '221325', 'license': 'MIT', 'n_tokens_mistral': 4604, 'n_tokens_neox': 4259, 'n_words': 701, 'path': 'Housing Prices.ipynb', 'repo_name': 'ayushmenon/home-prices', 'size': '12022'}