<jupyter_start><jupyter_text>Feature engineering- handle missing data- convert important categorical features- reduce the number of features (we only have about 1400 data points, cannot have too many features)<jupyter_code>n_na=[]
for col in data.columns[:-1]:
    n_na.append((sum(data_train[col].isna()), sum(data_test[col].isna()), data[col].dtype))

n_Na = list(zip(data.columns, n_na))

corrs = [(col, abs(data_train.SalePrice.corr(data_train[col]))) for col in data_train.columns[:-1] if data_train[col].dtype in ['int64','float64','bool']]
corrs.sort(key= lambda x:x[1], reverse=True)

corrs = dict(corrs)
corrs
display(n_Na)
# MSSubClass should be categorical, make dtype = str, so that can apply 
# pd.get_dummies later


data['MSSubClass'] = data['MSSubClass'].apply(str)
data['YrSold'] = data.YrSold.astype('str')
# exam the 1 missing value for 'electrical'
data.loc[data.Electrical.isna(), 'Electrical'] = data.Electrical.mode()
data.Electrical.value_counts()
# fill in missing 'LotFrontage' with the mode within house's neighborhood
def fillna_with_grouped_mode(col, groupby, data=data):
    """
    input: 
    col : column name to fill in NaN
    
    groupby: column name to groupby. 
    Modes are calculated within each group 
    and fill to the NaN value of that group.
    
    data = the dataframe contains col and groupby (default: data)
    """
    print(data[col].value_counts(dropna=False))

    for name, group in data[data[col].isna()].groupby(groupby):
        zone = data[data[groupby]==name][col].mode()[0]
        data.loc[group.index, col] = data.loc[group.index, col].fillna(zone)
    
    print('Filled NaN with nbhd mode:')
    print(data[col].value_counts(dropna=False))
fillna_with_grouped_mode('LotFrontage','Neighborhood', data)
fillna_with_grouped_mode('MSZoning','Neighborhood', data)
data['Alley'] = data.Alley.fillna('No')
data.Utilities.value_counts()
data = data.drop('Utilities', axis=1)
data.Exterior1st.value_counts(dropna=False)
data.Exterior2nd.value_counts()
data['MasVnrArea']=data['MasVnrArea'].fillna(0)
data[data.BsmtExposure.isna()| data.BsmtFinType2.isna()][['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinSF1','BsmtFinType2','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF']].loc[333,:]
data.loc[333,'BsmtExposure'] = np.nan
data[['BsmtFinSF1', 'BsmtFinSF2']] =data[['BsmtFinSF1', 'BsmtFinSF2']].fillna(0)
data.BsmtFinSF2.value_counts(dropna=False)
data.Electrical.isna().sum()
data.Electrical.value_counts(dropna=False)
data['Electrical']=data['Electrical'].fillna('SBrkr')
data['BsmtFullBath']=data['BsmtFullBath'].fillna(0)
data['BsmtHalfBath']=data['BsmtHalfBath'].fillna(0)
data.BsmtHalfBath.isna().sum()
data.loc[data.KitchenQual.isna(),'KitchenQual']=data.KitchenQual.mode()[0]
data.KitchenQual.mode()
data[data.KitchenQual.isna()]
data.Functional.value_counts(dropna=False)
data['Functional']=data['Functional'].fillna('Typ')
data.FireplaceQu.value_counts(dropna=False)
data.Fireplaces.value_counts(dropna=False)
data['GarageArea']=data['GarageArea'].fillna(0)
data.GarageType.value_counts(dropna=False)
data_test[(~data.GarageType.isna()) & data.GarageYrBlt.isna()][['GarageType','GarageYrBlt']]
data_test.loc[(~data.GarageType.isna()) & data.GarageYrBlt.isna(),'GarageYrBlt']=data_test.loc[(~data.GarageType.isna()) & data.GarageYrBlt.isna(),'YearRemodAdd']
data['SaleType']=data['SaleType'].fillna(data.SaleType.mode()[0])
data.SaleType.isna().sum()<jupyter_output><empty_output><jupyter_text>get_dummies variables<jupyter_code>cat_vars = [col for col in data.columns if data[col].dtype not in ['int64','float64','bool']]
cat_vars
data.shape
data = pd.get_dummies(data, prefix=cat_vars)
data.shape
corr = data.corr()
plt.figure(figsize=(20,16))
ax = sns.heatmap(corr, vmin=-1, vmax=1, center=0, cmap=sns.diverging_palette(20, 220, n=400))
corrs = [(col, abs(data[~data.SalePrice.isna()].SalePrice.corr(data[~data.SalePrice.isna()][col]))) for col in data.columns[:-1] if data[col].dtype in ['int64','float64','bool']]
corrs.sort(key= lambda x:x[1], reverse=True)

corrs = dict(corrs)
corrs
data_train_clean = data[~data.SalePrice.isna()]
plt.scatter(data_train_clean.OverallQual, data_train_clean.SalePrice)
plt.scatter(data_train_clean.GrLivArea, data_train_clean.SalePrice)
data_train_clean[(data_train_clean.GrLivArea>4000) & (data_train_clean.SalePrice<300000)]
data = data.drop([524,1299])
data_train_clean = data[~data.SalePrice.isna()]
plt.scatter(data_train_clean.GrLivArea, data_train_clean.SalePrice)
plt.scatter(data_train_clean.OverallQual, data_train_clean.SalePrice)<jupyter_output><empty_output><jupyter_text>prepare data for xgboost model<jupyter_code>#check that all data points are of numeric types
data.dtypes.value_counts()
y_train = data[~data.SalePrice.isna()]['SalePrice'].copy()
len(y_train)== len(data_train)
x_test = data[data.SalePrice.isna()]
x_train = data[~data.SalePrice.isna()]
len(x_test) == len(data_test), len(x_train) == len(data_train)

del x_train['SalePrice']
del x_test['SalePrice']
y_train.shape, x_train.shape, x_test.shape
from sklearn.model_selection import train_test_split
x_train, x_cv, y_train, y_cv = train_test_split(x_train, y_train, test_size=0.35,random_state =57)

import xgboost as xgb
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error as mse
DM_train = xgb.DMatrix(x_train.values, label=y_train)
DM_cv = xgb.DMatrix(x_cv, label=y_cv)
model_xgb = XGBRegressor(learning_rate=.03, n_estimators = 30000, max_depth = 2)

model_xgb.fit(x_train, y_train)
pred_cv=model_xgb.predict(x_cv)
pred_train = model_xgb.predict(x_train)

rmse_train = np.sqrt(mse(pred_train, y_train))
rmse_cv = np.sqrt(mse(pred_cv, y_cv))
rmses.append((rmse_train, rmse_cv))

print(rmses)<jupyter_output>[(6442.601552412794, 29898.42983599537), (1.2037702378608781, 32420.897235517823), (34.5018421168965, 30903.760048021966), (40.06785318035115, 30873.37158922885), (4738.01928140834, 30402.886635421797), (2894.449949301771, 31362.35584291857), (2844.198477517944, 26239.381879744124), (7.790203286891832, 22510.10525494061), (365.70566982678656, 22475.75446761173)]
{'fraction_non_alphanumeric': 0.15950114867082377, 'fraction_numerical': 0.06334099113882508, 'mean_word_length': 7.757183908045977, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '103583', 'license': 'Unlicense', 'n_tokens_mistral': 2693, 'n_tokens_neox': 2417, 'n_words': 360, 'path': 'notebook 3.ipynb', 'repo_name': 'hchen3368/Home-Price-Prediction', 'size': '6094'}