<jupyter_start><jupyter_text>test, train concat<jupyter_code>test["SalePrice"] = np.nan
house = pd.concat([train, test])
house.reset_index(inplace=True, drop=True)
house.shape<jupyter_output><empty_output><jupyter_text>data 전처리<jupyter_code>house.tail()
qual_df = pd.Series([50, 40, 30, 20, 10, 0], index=["Ex", "Gd", "TA", "Fa", "Po", "NA"])
qual_df<jupyter_output><empty_output><jupyter_text>LotFrontage - NaN 값 채우기<jupyter_code>print(len(house[house.LotFrontage.isna()]))
print(len(house))
house[house.LotFrontage.notna()].tail()
plt.scatter(house[house.LotFrontage.notna()].LotFrontage, house[house.LotFrontage.notna()].LotArea)
plt.show()
df = house[house.LotFrontage.notna()]
imp_df = house[house.LotFrontage.isna()]
# imp_df = df[df.LotArea < 100000]
corr = np.corrcoef(df.LotFrontage, df.LotArea)[1][0] * 100
print("correlation: {}%".format(round(corr, 2)))

corr = np.corrcoef(df[df.LotArea < 100000].LotFrontage, df[df.LotArea < 100000].LotArea)[1][0] * 100
print("correlation: {}%".format(round(corr, 2)))
# for i in range(1, len(temp.columns)):
#     if temp.columns[i] != 'SalePrice':
#         corr = np.corrcoef(temp.LotFrontage, temp.columns[i])[1][0] * 100
#         print("{} correlation: {}%".format(temp.columns[i], round(corr, 2)))<jupyter_output><empty_output><jupyter_text>Street<jupyter_code>house.drop(columns=["Street"], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Utilities<jupyter_code>house.drop(columns=["Utilities"], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Condition1<jupyter_code>mask = house.Condition1 != 'Norm'
house.loc[mask, 'Condition1'] = 'Other'
del mask<jupyter_output><empty_output><jupyter_text>Condition2<jupyter_code>mask = house.Condition2 != 'Norm'
house.loc[mask, 'Condition2'] = 'Other'
del mask<jupyter_output><empty_output><jupyter_text>RoofMatl<jupyter_code>house.drop(columns=["RoofMatl"], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Exterior1st<jupyter_code>house["Exterior1st"].fillna("None", inplace=True)
key = house.Exterior1st.value_counts() < 10
mask = house["Exterior1st"].map(key)
house.loc[mask, 'Exterior1st'] = 'Other'
del key, mask
# indices = list(house.Exterior1st[(house.Exterior1st == "BrkComm")| (house.Exterior1st == "AsphShn") | (house.Exterior1st == "Stone") | (house.Exterior1st == "CBlock") | (house.Exterior1st == "None") | (house.Exterior1st == "ImStucc")].index)
# for index in indices:
#     house.loc[index, 'Exterior1st'] = "other"
house.Exterior1st.value_counts()<jupyter_output><empty_output><jupyter_text>Exterior2nd<jupyter_code>house["Exterior2nd"].fillna("None", inplace=True)
key = house.Exterior2nd.value_counts() < 10
mask = house["Exterior2nd"].map(key)
house.loc[mask, 'Exterior2nd'] = 'Other'
del key, mask
house.Exterior2nd.value_counts()<jupyter_output><empty_output><jupyter_text>MasVnrType<jupyter_code># fill NaN 
house["MasVnrType"].fillna("None", inplace=True)<jupyter_output><empty_output><jupyter_text>MasVnrArea<jupyter_code># fill NaN 
house["MasVnrArea"].fillna(0, inplace=True)
house.tail()<jupyter_output><empty_output><jupyter_text>ExterQual<jupyter_code>house["ExterQual_emb"] = house["ExterQual"].map(qual_df).astype(int)
house.drop(columns=["ExterQual"], axis=1, inplace=True)
house.tail()<jupyter_output><empty_output><jupyter_text>ExterCond<jupyter_code>house["ExterCond_emb"] = house["ExterCond"].map(qual_df).astype(int)
house.drop(columns=["ExterCond"], axis=1, inplace=True)
house.tail()<jupyter_output><empty_output><jupyter_text>BsmtQual<jupyter_code># fill NaN 
house["BsmtQual"].fillna("NA", inplace=True)
house["BsmtQual_emb"] = house["BsmtQual"].map(qual_df).astype(int)
house.drop(columns=["BsmtQual"], axis=1, inplace=True)
house.tail()<jupyter_output><empty_output><jupyter_text>BsmtCond - NaN 값 처리 => no basement인 것으로 추정됨<jupyter_code># fill NaN 
house["BsmtCond"].fillna("NA", inplace=True)
house["BsmtCond_emb"] = house["BsmtCond"].map(qual_df).astype(int)
house.drop(columns=["BsmtCond"], axis=1, inplace=True)
house.tail()<jupyter_output><empty_output><jupyter_text>BsmtExposure<jupyter_code># fill NaN 
house.loc[house["Id"] == 949, 'BsmtExposure'] = "Av"
house["BsmtExposure"].fillna("NA", inplace=True)
Bsmt_df = pd.Series([40, 30, 20, 10, 0], index=["Gd", "Av", "Mn", "No", "NA"])
Bsmt_df
house["BsmtExposure_emb"] = house["BsmtExposure"].map(Bsmt_df).astype(int)
del Bsmt_df
house.drop(columns=["BsmtExposure"], axis=1, inplace=True)
house.tail()<jupyter_output><empty_output><jupyter_text>BsmtFinType1<jupyter_code># fill NaN 
house["BsmtFinType1"].fillna("NA", inplace=True)
Bsmt_df = pd.Series([60, 50, 40, 30, 20, 10, 0], index=["GLQ", "ALQ", "BLQ", "Rec", "LwQ", "Unf",  "NA"])
Bsmt_df
house["BsmtFinType1_emb"] = house["BsmtFinType1"].map(Bsmt_df).astype(int)
del Bsmt_df
house.drop(columns=["BsmtFinType1"], axis=1, inplace=True)
house.tail()<jupyter_output><empty_output><jupyter_text>BsmtFinSF1<jupyter_code># fill NaN 
house["BsmtFinSF1"].fillna(0, inplace=True)<jupyter_output><empty_output><jupyter_text>BsmtFinType2<jupyter_code># fill NaN 
house["BsmtFinType2"].fillna("NA", inplace=True)
Bsmt_df = pd.Series([60, 50, 40, 30, 20, 10, 0], index=["GLQ", "ALQ", "BLQ", "Rec", "LwQ", "Unf",  "NA"])
Bsmt_df
house["BsmtFinType2_emb"] = house["BsmtFinType2"].map(Bsmt_df).astype(int)
del Bsmt_df
house.drop(columns=["BsmtFinType2"], axis=1, inplace=True)
house.tail()<jupyter_output><empty_output><jupyter_text>BsmtFinSF2<jupyter_code># fill NaN 
house["BsmtFinSF2"].fillna(0, inplace=True)<jupyter_output><empty_output><jupyter_text>BsmtUnfSF<jupyter_code># fill NaN 
house["BsmtUnfSF"].fillna(0, inplace=True)<jupyter_output><empty_output><jupyter_text>TotalBsmtSF<jupyter_code># fill NaN 
house["TotalBsmtSF"].fillna(0, inplace=True)<jupyter_output><empty_output><jupyter_text>HeatingQC<jupyter_code>house["HeatingQC_emb"] = house["HeatingQC"].map(qual_df).astype(int)
house.drop(columns=["HeatingQC"], axis=1, inplace=True)
house.tail()<jupyter_output><empty_output><jupyter_text>Electrical<jupyter_code># fill NaN 
house["Electrical"].fillna("SBrkr", inplace=True)
house.loc[house["Electrical"] == "Mix", 'Electrical'] = "FuseP"
electrical_df = pd.Series([40, 30, 20, 10], index=["SBrkr", "FuseA", "FuseF", "FuseP"])
electrical_df
house["Electrical_emb"] = house["Electrical"].map(electrical_df).astype(int)
del electrical_df
house.drop(columns=["Electrical"], axis=1, inplace=True)
house.tail()<jupyter_output><empty_output><jupyter_text>BsmtFullBath<jupyter_code>test[test.BsmtHalfBath.isnull()]
# fill NaN 
house["BsmtFullBath"].fillna(0, inplace=True)
# BsmtFullBath_df = house["BsmtFullBath"].value_counts()
# house["BsmtFullBath_emb"] = house["BsmtFullBath"].map(BsmtFullBath_df).astype(int)
# house.drop(columns=["BsmtFullBath"], axis=1, inplace=True)
# house.tail()
# del BsmtFullBath_df
# show_feature_num(house, "BsmtFullBath_emb")<jupyter_output><empty_output><jupyter_text>BsmtHalfBath<jupyter_code># fill NaN 
house["BsmtHalfBath"].fillna(0, inplace=True)
# BsmtHalfBath_df = house["BsmtHalfBath"].value_counts()
# house["BsmtHalfBath_emb"] = house["BsmtHalfBath"].map(BsmtHalfBath_df).astype(int)
# house.drop(columns=["BsmtHalfBath"], axis=1, inplace=True)
# house.tail()
# del BsmtHalfBath_df
# show_feature_num(house, "BsmtHalfBath_emb")<jupyter_output><empty_output><jupyter_text>FullBath<jupyter_code># FullBath_df = house["FullBath"].value_counts()
# house["FullBath_emb"] = house["FullBath"].map(FullBath_df).astype(int)
# house.drop(columns=["FullBath"], axis=1, inplace=True)
# house.tail()
# del FullBath_df
# show_feature_num(house, "FullBath_emb")<jupyter_output><empty_output><jupyter_text>HalfBath<jupyter_code># HalfBath_df = house["HalfBath"].value_counts()
# house["HalfBath_emb"] = house["HalfBath"].map(HalfBath_df).astype(int)
# house.drop(columns=["HalfBath"], axis=1, inplace=True)
# house.tail()
# del HalfBath_df
# show_feature_num(house, "HalfBath_emb")<jupyter_output><empty_output><jupyter_text>BedroomAbvGr<jupyter_code># BedroomAbvGr_df = house["BedroomAbvGr"].value_counts()
# house["BedroomAbvGr_emb"] = house["BedroomAbvGr"].map(BedroomAbvGr_df).astype(int)
# house.drop(columns=["BedroomAbvGr"], axis=1, inplace=True)
# house.tail()
# del BedroomAbvGr_df
# show_feature_num(house, "BedroomAbvGr_emb")<jupyter_output><empty_output><jupyter_text>KitchenAbvGr<jupyter_code># KitchenAbvGr_df = house["KitchenAbvGr"].value_counts()
# house["KitchenAbvGr_emb"] = house["KitchenAbvGr"].map(KitchenAbvGr_df).astype(int)
# house.drop(columns=["KitchenAbvGr"], axis=1, inplace=True)
# house.tail()
# del KitchenAbvGr_df
# show_feature_num(house, "KitchenAbvGr_emb")<jupyter_output><empty_output><jupyter_text>KitchenQual<jupyter_code># fill NaN 
house.KitchenQual.fillna("NA", inplace=True)
house["KitchenQual_emb"] = house["KitchenQual"].map(qual_df).astype(int)
house.drop(columns=["KitchenQual"], axis=1, inplace=True)
# house.tail()<jupyter_output><empty_output><jupyter_text>TotRmsAbvGrd<jupyter_code># TotRmsAbvGrd_df = house["TotRmsAbvGrd"].value_counts()
# house["TotRmsAbvGrd_emb"] = house["TotRmsAbvGrd"].map(TotRmsAbvGrd_df).astype(int)
# house.tail()
# del TotRmsAbvGrd_df
# show_feature_num(house, "TotRmsAbvGrd_emb")<jupyter_output><empty_output><jupyter_text>Fireplaces<jupyter_code>fireplaces_df = house["Fireplaces"].value_counts()
house["Fireplaces_emb"] = house["Fireplaces"].map(fireplaces_df).astype(int)
del fireplaces_df
# house.tail()<jupyter_output><empty_output><jupyter_text>FireplaceQu<jupyter_code># fill NaN 
house.FireplaceQu.fillna("NA", inplace=True)
house["FireplaceQu_emb"] = house["FireplaceQu"].map(qual_df).astype(int)
house.drop(columns=["FireplaceQu"], axis=1, inplace=True)
# house.tail()<jupyter_output><empty_output><jupyter_text>GarageType<jupyter_code># fill NaN 
house.GarageType.fillna("NA", inplace=True)<jupyter_output><empty_output><jupyter_text>GarageYrBlt<jupyter_code># fill NaN 
house.GarageYrBlt.fillna(0, inplace=True)<jupyter_output><empty_output><jupyter_text>GarageFinish<jupyter_code># fill NaN 
house.GarageFinish.fillna("NA", inplace=True)
garage_df = pd.Series([30, 20, 10, 0], index=["Fin", "RFn", "Unf", "NA"])
garage_df
house["GarageFinish_emb"] = house["GarageFinish"].map(garage_df).astype(int)
del garage_df
house.drop(columns=["GarageFinish"], axis=1, inplace=True)
# house.tail()<jupyter_output><empty_output><jupyter_text>GarageCars<jupyter_code># fill NaN 
house.GarageCars.fillna(0, inplace=True)<jupyter_output><empty_output><jupyter_text>GarageArea<jupyter_code># fill NaN 
house.GarageArea.fillna(0, inplace=True)<jupyter_output><empty_output><jupyter_text>GarageQual<jupyter_code># fill NaN 
house.GarageQual.fillna("NA", inplace=True)
house["GarageQual_emb"] = house["GarageQual"].map(qual_df).astype(int)
house.drop(columns=["GarageQual"], axis=1, inplace=True)
# house.tail()<jupyter_output><empty_output><jupyter_text>GarageCond<jupyter_code># fill NaN 
house.GarageCond.fillna("NA", inplace=True)
house["GarageCond_emb"] = house["GarageCond"].map(qual_df).astype(int)
house.drop(columns=["GarageCond"], axis=1, inplace=True)
# house.tail()<jupyter_output><empty_output><jupyter_text>PavedDrive<jupyter_code>paved_df = pd.Series([30, 20, 10], index=["Y", "P", "N"])
paved_df
house["PavedDrive_emb"] = house["PavedDrive"].map(paved_df).astype(int)
del paved_df
house.drop(columns=["PavedDrive"], axis=1, inplace=True)
# house.tail()<jupyter_output><empty_output><jupyter_text>PoolArea<jupyter_code>house.drop(columns=["PoolArea"], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>PoolQC<jupyter_code># fill NaN 
house.PoolQC.fillna("NA", inplace=True)
house["PoolQC_emb"] = house["PoolQC"].map(qual_df).astype(int)
house.drop(columns=["PoolQC"], axis=1, inplace=True)
# house.tail()<jupyter_output><empty_output><jupyter_text>Fence feature 2개로 분리- Fence 종류 category feature (Privacy / Wood / No Fence)- Fence quality numeric feature (Good / Minimum) => data embeding<jupyter_code># fill NaN 
house.Fence.fillna("NA", inplace=True)
fence_type = pd.Series(["Prv", "Prv", "Ww", "Ww", "NA"], index=["GdPrv", "MnPrv", "GdWo", "MnWw", "NA"])
fence_type

house["FenceType"] = house["Fence"].map(fence_type)
del fence_type
fence_qual = pd.Series([30, 20, 30, 20, 10], index=["GdPrv", "MnPrv", "GdWo", "MnWw", "NA"])
fence_qual

house["FenceQual"] = house["Fence"].map(fence_qual).astype(int)
del fence_qual
house.drop(columns=["Fence"], axis=1, inplace=True)
# house.tail()<jupyter_output><empty_output><jupyter_text>MiscFeature<jupyter_code>house.drop(columns=["MiscFeature"], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>MoSold<jupyter_code>MoSold_count = house["MoSold"].value_counts()
house["MoSold_emb"] = house["MoSold"].map(MoSold_count).astype(float)
del MoSold_count
house.drop(columns=["MoSold"], axis=1, inplace=True)
# house.tail()<jupyter_output><empty_output><jupyter_text>YrSold<jupyter_code>house.drop(columns=["YrSold"], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>data frame<jupyter_code>house.columns
get_dummy = ['MSSubClass', 'MSZoning', 'Alley', 'LotShape', 'LandContour', 'LotConfig',
             'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle',
             'RoofStyle', 'RoofMatl','Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation',
             'Heating','CentralAir', 'Functional', 'GarageType', 'SaleType', 'SaleCondition', 
             'FenceType']
temp = house.filter(get_dummy)

dummy_df = pd.get_dummies(temp[temp.columns[0]])
dummy_df.columns = ["MSSubClass_" + str(col) for col in dummy_df.columns]

for num in range(1, len(temp.columns)):
    dummy = pd.get_dummies(temp[temp.columns[num]])
    dummy.columns = [str(temp.columns[num]) + "_" + str(col) for col in dummy.columns]
    dummy_df = pd.concat([dummy_df, dummy], axis=1)
del temp
get_num = list(set(house.columns) - set(get_dummy))
numeric_df = house.filter(get_num)
df = pd.concat([numeric_df, dummy_df], axis=1)
df.tail()
df.to_csv("csv/df.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17994289793004997, 'fraction_numerical': 0.014989293361884369, 'mean_word_length': 10.608119304059652, 'pattern_counts': {'":': 0, '<': 206, '<?xml version=': 0, '>': 203, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '50', 'dir': 'jupyter-structured-clean-dedup', 'id': '396721', 'license': 'MIT', 'n_tokens_mistral': 6059, 'n_tokens_neox': 5799, 'n_words': 656, 'path': '20180508_Feature_preprocessing.ipynb', 'repo_name': 'hoophoop111/house_price', 'size': '14010'}