<jupyter_start><jupyter_text>Intro We use the [Kaggle Dataset "House Prices"](https://www.kaggle.com/c/house-prices-advanced-regression-techniques/overview) to show an example how to use ML Aide.<jupyter_code>## for data
import pandas as pd
pd.options.mode.chained_assignment = None  # default='warn'
import numpy as np

## for plotting
import matplotlib.pyplot as plt
import seaborn as sns

## for statistical tests
import scipy
import statsmodels.formula.api as smf
import statsmodels.api as sm

## for machine learning
from sklearn import model_selection, preprocessing, feature_selection, ensemble, linear_model, metrics, decomposition
dtf = pd.read_csv("data/data_houses.csv")
cols = ["OverallQual","GrLivArea","GarageCars", 
        "GarageArea","TotalBsmtSF","FullBath",
        "YearBuilt","YearRemodAdd",
        "LotFrontage","MSSubClass"]
dtf = dtf[["Id"] + cols + ["SalePrice"]]

dtf.head()
def utils_recognize_type(dtf, col, max_cat=20):
    if (dtf[col].dtype == "O") | (dtf[col].nunique() < max_cat):
        return "cat"
    else:
        return "num"<jupyter_output><empty_output><jupyter_text>Data Visualization<jupyter_code>dic_cols = {col:utils_recognize_type(dtf, col, max_cat=20) for col in dtf.columns}
heatmap = dtf.isnull()
for k,v in dic_cols.items():
  if v == "num":
    heatmap[k] = heatmap[k].apply(lambda x: 0.5 if x is False else 1)
  else:
    heatmap[k] = heatmap[k].apply(lambda x: 0 if x is False else 1)
sns.heatmap(heatmap, cbar=False).set_title('Dataset Overview')
plt.show()
print("\033[1;37;40m Categerocial ", "\033[1;30;41m Numeric ", "\033[1;30;47m NaN ")
dtf = dtf.set_index("Id")
dtf = dtf.rename(columns={"SalePrice":"Y"})
x = "Y"
fig, ax = plt.subplots(nrows=1, ncols=2,  sharex=False, sharey=False)
fig.suptitle(x, fontsize=20)
### distribution
ax[0].title.set_text('distribution')
variable = dtf[x].fillna(dtf[x].mean())
breaks = np.quantile(variable, q=np.linspace(0, 1, 11))
variable = variable[ (variable > breaks[0]) & (variable < 
                    breaks[10]) ]
sns.distplot(variable, hist=True, kde=True, kde_kws={"shade": True}, ax=ax[0])
des = dtf[x].describe()
ax[0].axvline(des["25%"], ls='--')
ax[0].axvline(des["mean"], ls='--')
ax[0].axvline(des["75%"], ls='--')
ax[0].grid(True)
des = round(des, 2).apply(lambda x: str(x))
box = '\n'.join(("min: "+des["min"], "25%: "+des["25%"], "mean: "+des["mean"], "75%: "+des["75%"], "max: "+des["max"]))
ax[0].text(0.95, 0.95, box, transform=ax[0].transAxes, fontsize=10, va='top', ha="right", bbox=dict(boxstyle='round', facecolor='white', alpha=1))
### boxplot 
ax[1].title.set_text('outliers (log scale)')
tmp_dtf = pd.DataFrame(dtf[x])
tmp_dtf[x] = np.log(tmp_dtf[x])
tmp_dtf.boxplot(column=x, ax=ax[1])
plt.show()<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code>## split data
dtf_train, dtf_test = model_selection.train_test_split(dtf, test_size=0.3)

## print info
print("X_train shape:", dtf_train.drop("Y",axis=1).shape, "| X_test shape:", dtf_test.drop("Y",axis=1).shape)
print("y_train mean:", round(np.mean(dtf_train["Y"]),2), "| y_test mean:", round(np.mean(dtf_test["Y"]),2))
print(dtf_train.shape[1], "features:", dtf_train.drop("Y",axis=1).columns.to_list())<jupyter_output>X_train shape: (1022, 10) | X_test shape: (438, 10)
y_train mean: 179430.18 | y_test mean: 184400.24
11 features: ['OverallQual', 'GrLivArea', 'GarageCars', 'GarageArea', 'TotalBsmtSF', 'FullBath', 'YearBuilt', 'YearRemodAdd', 'LotFrontage', 'MSSubClass']<jupyter_text>Fill missing LotFrontage values with the mean<jupyter_code>dtf_train["LotFrontage"] = dtf_train["LotFrontage"].fillna(dtf_train["LotFrontage"].mean())<jupyter_output><empty_output><jupyter_text>Encode categorical data<jupyter_code>## create dummy
dummy = pd.get_dummies(dtf_train["MSSubClass"], prefix="MSSubClass_cluster", drop_first=True)
dtf_train= pd.concat([dtf_train, dummy], axis=1)
print(dtf_train.filter(like="MSSubClass",axis=1).head())

## drop the original categorical column
dtf_train = dtf_train.drop("MSSubClass", axis=1)<jupyter_output>MSSubClass  MSSubClass_cluster_30  MSSubClass_cluster_40  \
Id                                                               
862          190                      0                      0   
952           20                      0                      0   
1173         160                      0                      0   
1135          60                      0                      0   
1155          60                      0                      0   

      MSSubClass_cluster_45  MSSubClass_cluster_50  MSSubClass_cluster_60  \
Id                                                                          
862                       0                      0                      0   
952                       0                      0                      0   
1173                      0                      0                      0   
1135                      0                      0                      1   
1155                      0                      0                      1  [...]<jupyter_text>Scale features<jupyter_code>## scale X
scalerX = preprocessing.RobustScaler(quantile_range=(25.0, 75.0))
X = scalerX.fit_transform(dtf_train.drop("Y", axis=1))
dtf_scaled= pd.DataFrame(X, columns=dtf_train.drop("Y", axis=1).columns, index=dtf_train.index)
## scale Y
scalerY = preprocessing.RobustScaler(quantile_range=(25.0, 75.0))
dtf_scaled[y] = scalerY.fit_transform(dtf_train[y].values.reshape(-1,1))
dtf_scaled.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15265928480667998, 'fraction_numerical': 0.03902704665093484, 'mean_word_length': 2.9927536231884058, 'pattern_counts': {'":': 3, '<': 25, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '647242', 'license': 'Apache-2.0', 'n_tokens_mistral': 2088, 'n_tokens_neox': 1930, 'n_words': 368, 'path': 'examples/house_prices/Untitled.ipynb', 'repo_name': 'MLAide/python-client', 'size': '5509'}