<jupyter_start><jupyter_text>Types of data:- numerical    - discrete (e.g. number of bathrooms, age)    - continuous (e.g. temperature, weight)- categorical (e.g. color, type of material)- ordinal (e.g. rating, university grade)<jupyter_code>interval_columns = ['MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual', 
                   'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 
                   'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 
                   '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 
                   'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 
                   'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 
                   'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF', 
                   'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 
                   'PoolArea', 'MiscVal', 'MoSold', 'YrSold', 'SalePrice']
fake_interval_columns = ['MSSubClass', 'OverallQual', 'OveralCond', 'MoSold']

interval_columns = list(set(interval_columns) - set(fake_interval_columns))
# Calculate "correlation" between features and target
corr = data.phik_matrix(interval_cols=interval_columns)
fig, ax = plt.subplots(figsize=(20, 20), facecolor='w')
sns.heatmap(corr, ax=ax, vmin=-1, vmax=1, cmap='RdBu_r')<jupyter_output><empty_output><jupyter_text>Show only correlations between SalePrice (target) and all other features.<jupyter_code>corr_price = corr["SalePrice"]

# Show features with highest correlation with price
corr_price = corr_price.sort_values(ascending=False)
corr_price.head(5)<jupyter_output><empty_output><jupyter_text>Remove unimportant features<jupyter_code>correlation_threshold = 0.3
features_to_keep = corr_price[corr_price > correlation_threshold].index

# Remove features weakly correlating with target 
data = data[features_to_keep]
data.shape

data_filtered = data.copy()
data_filtered.shape<jupyter_output><empty_output><jupyter_text>Deal with categorical data<jupyter_code>non_interval_columns = set(data.columns) - set(interval_columns)
data[non_interval_columns].dtypes

ordinal_columns = ['HeatingQC', 'GarageQual', 'BsmtFinType1', 'ExterQual', 
                   'GarageFinish', 'BsmtExposure', 'LotShape', 'OverallQual',
                   'BsmtQual', 'KitchenQual']
categorical_columns = list(set(non_interval_columns) - set(ordinal_columns))
# Use One Hot Encoding for categorical columns
for column in categorical_columns: 
    data = pd.get_dummies(data, columns=[column], prefix=column, prefix_sep='_', 
                          drop_first=True)
# Change this to remove categories with less than dummies_theshold percent 
# representatives
dummies_threshold = 0.1

# Find binary features (with two possible values 0 and 1) that have less than
# dummies_threshold ons
binary_columns = [set(data[column].unique()) == {0, 1} for column in data]
binary_columns_to_drop = data.loc[:, binary_columns].mean() < dummies_threshold
binary_columns_to_drop = list(binary_columns_to_drop[binary_columns_to_drop].index)

print(f'Dropped {len(binary_columns_to_drop)} columns with less than ' + \
      f'{dummies_threshold*100}% representatives')

data = data.drop(binary_columns_to_drop, axis=1)
# Filter numerical data for visualisation purpose
df = data.select_dtypes(['number'])
df = 1 - ((df.max() - df) / df.max())

# Display numerical data
fig, ax = plt.subplots(figsize=(30, 30), facecolor='w')
im = ax.imshow(df.values, aspect='auto', interpolation='none', cmap='gray')
plt.colorbar(im);<jupyter_output><empty_output><jupyter_text>Deal with ordinal data<jupyter_code>for c in data[ordinal_columns]:
    print(c, data[c].unique())
# Using def syntax
def nan_column(df, column):
    '''Creates column with ones and zeros encoding rows with nan.'''
    return (~df[column].isna()).astype(int)

for column in ordinal_columns:
    if data[column].isna().any():
        print(f'Creating nan-encoding column for {column}')
        data[f'is{column}'] = nan_column(data, column)<jupyter_output>Creating nan-encoding column for GarageQual
Creating nan-encoding column for BsmtFinType1
Creating nan-encoding column for GarageFinish
Creating nan-encoding column for BsmtExposure
Creating nan-encoding column for BsmtQual<jupyter_text>> Tip: filter out columns starting with string 'is'```pythondata.loc[:, data.columns.str.startswith('is')]``` ```BsmtExposure: Refers to walkout or garden level walls       Gd	Good Exposure       Av	Average Exposure (split levels or foyers typically score average or above)	       Mn	Mimimum Exposure       No	No Exposure       NA	No Basement```<jupyter_code>def mapping_from_list(order):
    return {label: idx for idx, label in enumerate(order)}

# Features with ordinal data having same levels (Poor to Excelent)
quality_mapping = mapping_from_list([np.nan, 'Po', 'Fa', 'TA', 'Gd', 'Ex'])
quality_mapping_columns = ['HeatingQC', 'GarageQual', 'ExterQual', 'BsmtQual',
                           'KitchenQual']

for column in quality_mapping_columns :
    data[column] = data[column].map(quality_mapping)

# Other features with ordinal data having weird levels :D
data['LotShape'] = data['LotShape'].map(
    mapping_from_list([np.nan, 'IR3', 'IR2', 'IR1', 'Reg']))

data['BsmtFinType1'] = data['BsmtFinType1'].map(
    mapping_from_list([np.nan, 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']))

data['GarageFinish'] = data['GarageFinish'].map(
    mapping_from_list([np.nan, 'Unf', 'RFn', 'Fin']))

data['BsmtExposure'] = data['BsmtExposure'].map(
    mapping_from_list([np.nan, 'No', 'Mn', 'Av', 'Gd']))
data.select_dtypes(['number']).shape == data.shape
# Find numerical columns with remaining nan's 
nans_left = data.isna().sum() > 0
print(nans_left[nans_left])

data['MasVnrArea'] = data['MasVnrArea'].fillna(0)

# Remove this feature because it is strongly correlated with YearBuilt
data = data.drop('GarageYrBlt', axis=1)
plt.figure(figsize=(15, 6), facecolor='w')
sns.scatterplot(
    x='LotFrontage',
    y='SalePrice',
    data=data,
)

# Impute nan's with mean, because we cannot assume 0 distance from street
data['LotFrontage'] = data['LotFrontage'].fillna(data['LotFrontage'].mean())<jupyter_output><empty_output><jupyter_text>Remove highly correlated features<jupyter_code>plt.style.use('default')
fig, ax = plt.subplots(figsize=(6, 6), facecolor='w')
corr = data.corr()
im = ax.imshow(corr, cmap='RdBu_r', clim=[-1, 1])
plt.tight_layout()
# Remove highly correlated features
threshold_correlation = 0.8

upper = corr.abs().where(np.triu(np.ones(corr.shape), k=1).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.9)]

data.drop(to_drop, axis=1, inplace=True)
# Store data to train model
data.to_csv('data/train_clean.csv')
X = data.drop('SalePrice', axis=1)
y = data['SalePrice']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, 
                                                    random_state=100)

scaler = StandardScaler()
X_train_std = scaler.fit_transform(X_train)
X_test_std = scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>```python>>> import numpy as np>>> from sklearn.linear_model import LinearRegression>>> X = np.array([[1, 1], [1, 2], [2, 2], [2, 3]])>>>  y = 1 * x_0 + 2 * x_1 + 3>>> y = np.dot(X, np.array([1, 2])) + 3>>> reg = LinearRegression().fit(X, y)>>> reg.score(X, y)1.0>>> reg.coef_array([1., 2.])>>> reg.intercept_3.0000...>>> reg.predict(np.array([[3, 5]]))array([16.])```<jupyter_code>reg = LinearRegression()
reg.fit(X_train_std, y_train)
# Predicting price of single house ("by hand")
np.sum(reg.coef_ * X_test_std[0]) + reg.intercept_

# ... or using model method
reg.predict(X_test_std)[0], y_test.iloc[0]
r2_score(y_test, reg.predict(X_test_std))
plt.plot(y_test, reg.predict(X_test_std), 'ko')
plt.xlabel('Real price')
plt.ylabel('Predicted price')
plt.xlim([0, 600_000])
plt.ylim([0, 600_000])
plt.plot([0, 600_000], [0, 600_000], 'r')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1475759859331826, 'fraction_numerical': 0.015071590052750565, 'mean_word_length': 5.10191570881226, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '318284', 'license': 'MIT', 'n_tokens_mistral': 2933, 'n_tokens_neox': 2735, 'n_words': 611, 'path': 'house_prices/analysis06.ipynb', 'repo_name': 'randat9/House_Prices', 'size': '7962'}