<jupyter_start><jupyter_text>Checking distributions<jupyter_code>import scipy.stats as stats
fig, axes = plt.subplots(nrows=len(df.columns[1:]), figsize=(30, 50))

for ax, col in zip(axes, df.columns[1:]):
    sns.distplot(df[col], ax=ax)
   # plt.title(col)
    plt.tight_layout()
    
plt.show()<jupyter_output><empty_output><jupyter_text>Checking q-q plots<jupyter_code>stats.probplot(df['Age'], dist='norm', plot=plt)
plt.show()
stats.probplot(df['Income'], dist='norm', plot=plt)
plt.show()<jupyter_output><empty_output><jupyter_text>Checking for outliers<jupyter_code>plt.figure(figsize=(3, 6))
sns.boxplot(y=df['Age']);
plt.figure(figsize=(3, 6))
sns.boxplot(y=df['Income']);
def boundaries(df, column, dist):
    iqr = df[column].quantile(0.75) - df[column].quantile(0.25)
    
    lower = df[column].quantile(0.25) - iqr * dist
    upper = df[column].quantile(0.75) + iqr * dist
    
    return upper, lower
upper_age, lower_age = boundaries(df, 'Age', 1.5)
upper_age, lower_age
upper_income, lower_income = boundaries(df, 'Income', 1.5)
upper_income, lower_income<jupyter_output><empty_output><jupyter_text>Checking Data Magnitude<jupyter_code>df.max() - df.min()<jupyter_output><empty_output><jupyter_text>Correlation<jupyter_code>corr = df.corr()
sns.heatmap(corr, annot=True);<jupyter_output><empty_output><jupyter_text>Scaling<jupyter_code>from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
df_scaled = scaler.fit_transform(df)
df_scaled
df_scaled = pd.DataFrame(df_scaled, columns=df.columns.values)
df_scaled.head()
df_scaled.to_csv(data_folder + 'processed/segmentation_scaled.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15789473684210525, 'fraction_numerical': 0.01572897761645493, 'mean_word_length': 7.569948186528498, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '160745', 'license': 'MIT', 'n_tokens_mistral': 694, 'n_tokens_neox': 653, 'n_words': 89, 'path': 'notebooks/0.1-pre-eda.ipynb', 'repo_name': 'sidd-hart/customer-segmentation', 'size': '1653'}