<jupyter_start><jupyter_text>Load Data<jupyter_code>train = pd.read_csv('./data/train.csv')
test = pd.read_csv('./data/test.csv')
train_labels = train['SalePrice'] # separate labels from train dataset
data = pd.concat([train, test], keys=['train', 'test'])<jupyter_output><empty_output><jupyter_text>General Analysis<jupyter_code>print(data.columns) # check column decorations
print('rows:', data.shape[0], ', columns: ', data.shape[1]) # count rows of total dataset
print('rows in train dataset: ', train.shape[0])
print('rows in test dataset: ', test.shape[0])
nans = pd.concat([train.isnull().sum(), train.isnull().sum() / train.shape[0], test.isnull().sum(), test.isnull().sum()
                  / test.shape[0]], axis=1, keys=['Train', 'Percentage', 'Test', 'Percentage'])
print(nans[nans.sum(axis=1) > 0])<jupyter_output><empty_output><jupyter_text>Exploration on SalePrice<jupyter_code>print(train_labels.describe())
print("Skewness: %f" % train_labels.skew())
print("Kurtosis: %f" % train_labels.kurt())
plt.subplot(1, 2, 1)
plt.title("Sale Prices Dist")
sns.distplot(train_labels, fit=stats.norm)
plt.subplot(1, 2, 2)
stats.probplot(train_labels, plot=plt)
plt.show()
# log transformation
plt.subplot(1, 2, 1)
plt.title("Sale Prices Dist")
sns.distplot(np.log(train_labels), fit=stats.norm)
plt.subplot(1, 2, 2)
stats.probplot(np.log(train_labels), plot=plt)
plt.show()
print("Skewness: %f" % np.log(train_labels).skew())
print("Kurtosis: %f" % np.log(train_labels).kurt())<jupyter_output><empty_output><jupyter_text>Explore on Features<jupyter_code>train.drop(['Id', 'MiscFeature', 'Fence', 'PoolQC', 'FireplaceQu', 'Alley'], axis=1, inplace=True)
# draw correlation coefficient matrix
corrmat = train.corr()
f, ax = plt.subplots(figsize=(12, 9))
sns.heatmap(corrmat, vmax=.8, square=True, cmap="YlGnBu")
plt.yticks(rotation=0)
plt.xticks(rotation=90)
plt.show()
# number of variables for heatmap
corrmat = train.corr()
cols = corrmat.nlargest(10, 'SalePrice')['SalePrice'].index
cm = np.corrcoef(train[cols].values.T)
sns.set(font_scale=1.25)
hm = sns.heatmap(cm, cbar=True, annot=True, square=True, cmap="YlGnBu", fmt='.2f', annot_kws={'size': 10}, 
                 yticklabels=cols.values, xticklabels=cols.values)
plt.yticks(rotation=0)
plt.xticks(rotation=90)
plt.show()
# Bsmt relations
f, (ax1, ax2, ax3) = plt.subplots(1, 3)
data_total_bsmt = pd.concat([train['SalePrice'], train['TotalBsmtSF']], axis=1)
data_total_bsmt.plot.scatter(x='TotalBsmtSF', y='SalePrice', ylim=(0, 800000), ax=ax1)
data1 = pd.concat([train['SalePrice'], train['1stFlrSF']], axis=1)
data1.plot.scatter(x='1stFlrSF', y='SalePrice', ylim=(0, 800000), ax=ax2)
data2 = pd.concat([train['SalePrice'], train['2ndFlrSF']], axis=1)
data2.plot.scatter(x='2ndFlrSF', y='SalePrice', ylim=(0, 800000), ax=ax3)
plt.show()
sns.set()
cols = ['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'FullBath', 'YearBuilt']
sns.pairplot(train[cols], size=2.5)
plt.show()
train.drop(['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF'], axis=1, inplace=True)
train['TotalBsmtSF'] = train['TotalBsmtSF'].fillna(0)
train['1stFlrSF'] = train['1stFlrSF'].fillna(0)
train['2ndFlrSF'] = train['2ndFlrSF'].fillna(0)
train['TotalSF'] = train['TotalBsmtSF'] + train['1stFlrSF'] + train['2ndFlrSF']
train.drop(['TotalBsmtSF', '1stFlrSF', '2ndFlrSF'], axis=1, inplace=True)
train.drop(['GarageArea'], axis=1, inplace=True) # as analysis before
# draw TotalSF-SalePrice
data_total = pd.concat([train['SalePrice'], train['TotalSF']], axis=1)
data_total.plot.scatter(x='TotalSF', y='SalePrice', ylim=(0, 800000))
plt.show()
f, (ax1, ax2, ax3) = plt.subplots(1, 3)
data_gr = pd.concat([train['SalePrice'], train['GrLivArea']], axis=1)
data_gr.plot.scatter(x='GrLivArea', y='SalePrice', ylim=(0, 800000), ax=ax1)
data_to = pd.concat([train['SalePrice'], train['TotRmsAbvGrd']], axis=1)
data_to.plot.scatter(x='TotRmsAbvGrd', y='SalePrice', ylim=(0, 800000), ax=ax2)
gr_to = pd.concat([train['GrLivArea'], train['TotRmsAbvGrd']], axis=1)
gr_to.plot.scatter(x='TotRmsAbvGrd', y='GrLivArea', ax=ax3)
plt.show()
train.drop(['Utilities', 'RoofMatl', 'MasVnrArea', 'MasVnrType', 'Heating', 'LowQualFinSF',
            'BsmtFullBath', 'BsmtHalfBath', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
            'Functional', 'GarageYrBlt', 'GarageCond', 'GarageType', 'GarageFinish', 'GarageQual', 'WoodDeckSF',
            'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',
            'MiscVal'], axis=1, inplace=True)
print(train.columns)
# OverallQual and SalePrice
overall_qual = pd.concat([train['SalePrice'], train['OverallQual']], axis=1)
f, ax = plt.subplots(figsize=(8, 6))
fig = sns.boxplot(x='OverallQual', y="SalePrice", data=overall_qual)
fig.axis(ymin=0, ymax=800000)
plt.show()
# YearBuild and SalePrice
year_built = pd.concat([train['SalePrice'], train['YearBuilt']], axis=1)
f, ax = plt.subplots(figsize=(16, 8))
fig = sns.boxplot(x='YearBuilt', y="SalePrice", data=year_built)
fig.axis(ymin=0, ymax=800000)
plt.xticks(rotation=90)
plt.show()
plt.figure()
plt.subplot(1, 2, 1)
plt.title("GrLivArea Dist")
sns.distplot(train['GrLivArea'], fit=stats.norm)
plt.subplot(1, 2, 2)
stats.probplot(train['GrLivArea'], plot=plt)
plt.show()
print("Skewness: %f" % train['GrLivArea'].skew())
print("Kurtosis: %f" % train['GrLivArea'].kurt())
sf = np.log(train['GrLivArea'])
sp = np.log(train['SalePrice'])
plt.scatter(sf[sf > 0], sp[sf > 0])
plt.show()
# MSSubClass and SalePrice, and others
mssubclass = pd.concat([train['SalePrice'], train['ExterQual']], axis=1)
f, ax = plt.subplots(figsize=(8, 6))
fig = sns.boxplot(x='ExterQual', y="SalePrice", data=mssubclass)
fig.axis(ymin=0, ymax=800000)
plt.xticks(rotation=90)
plt.show()
# LotFrontage, LotArea and SalePrice, and others
# f, (ax1, ax2) = sns.plt.subplots(1, 2)
LotFrontage = pd.concat([train['SalePrice'], train['OverallCond']], axis=1)
LotFrontage.plot.scatter(x='OverallCond', y='SalePrice', ylim=(0, 800000))
# LotArea = pd.concat([train['SalePrice'], train['LotArea']], axis=1)
# LotArea.plot.scatter(x='LotArea', y='SalePrice', ylim=(0, 800000), ax=ax2)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2062874736671528, 'fraction_numerical': 0.03224760978771674, 'mean_word_length': 7.524861878453039, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '109837', 'license': 'MIT', 'n_tokens_mistral': 2672, 'n_tokens_neox': 2510, 'n_words': 413, 'path': 'house_prices_predict/data_analysis.ipynb', 'repo_name': 'IsaacChanghau/AmusingPythonCodes', 'size': '6171'}