<jupyter_start><jupyter_text>Principal Component Analysis<jupyter_code>from sklearn.preprocessing import Normalizer
normalizer = Normalizer()
normalized_values = normalizer.fit_transform(X)
normalized_values
%%time
from sklearn.decomposition import PCA

# One less as original feature number
N_COMPONENTS = clustering_df.shape[1] - 1


pca = PCA(n_components=N_COMPONENTS)
pca.fit(normalized_values)

print('explained_variance_ratio_ vector:', pca.explained_variance_ratio_)
print('explained_variance_ratio_ sum:', pca.explained_variance_ratio_.sum())


s = pd.DataFrame(pca.singular_values_)
v = pd.DataFrame(pca.components_)
f.explained_variance(s, 4)
figure = plt.figure(figsize=(10,10))

for i in range(4):
    plt.subplot(4,1,i+1)
    f.display_components(v, clustering_df.columns, i, n_weights=4)
    plt.title('Component #{}'.format(i+1))
figure.tight_layout()<jupyter_output><empty_output><jupyter_text>Setup K-MeansCreate K-Means analysis to find groups in customers based only on customer features  Create input data based on PCA analysisUse the `pca.transform()` function to create input data for the K-Means analysis.<jupyter_code>%%time
# Create input data from PCA Analysis
pca_transformed = pca.fit_transform(normalized_values)

pca_transformed = pd.DataFrame(pca_transformed)
pca_transformed = pca_transformed.iloc[:,:4]
PCA_list=['c_1', 'c_2', 'c_3', 'c_4']
pca_transformed.columns=PCA_list

pca_transformed.head()


%%time
from sklearn.cluster import KMeans
from scipy.spatial.distance import cdist


# Do a loop over different k's to find the optimum
inertias = []
distortions = []
for i in range(1,7):
    kmeans = KMeans(n_clusters=i).fit(pca_transformed)
    cluster_centers = kmeans.cluster_centers_
    inertias.append(kmeans.inertia_)
    
    distortions.append(sum(np.min(cdist(pca_transformed, kmeans.cluster_centers_,
                                        'euclidean'), axis=1)) / pca_transformed.shape[0])
                                        
    inertias.append(kmeans.inertia_)
    
distortions
sns.lineplot(x=range(1,7), y=distortions)
%%time
from sklearn.cluster import KMeans

kmeans = KMeans(n_clusters=5).fit(pca_transformed)
labels = kmeans.labels_
cluster_df = pd.DataFrame(kmeans.cluster_centers_, columns=pca_transformed.columns)

plt.figure(figsize = (12,9))
ax = sns.heatmap(cluster_df.T, cmap = 'YlGnBu', annot=True)
ax.set_xlabel("Cluster")
plt.yticks(fontsize = 16)
plt.xticks(fontsize = 16)
ax.set_title("Attribute Value by Centroid")


df.loc[:, 'label'] = labels
ticks = [0, 3, 4, 1, 2]
label_mapping = {0:'very high priced', 3:'high priced', 4:'normal priced', 1:'low priced', 2:'very low priced'}
df.label = df.label.map(label_mapping)

city_count = df.groupby(['label', ]).City.value_counts().reset_index(name='city_counts')
#city_count = city_count.pivot(index='City', columns='label', values='city_counts')
df.anzahletagen
city_count = df.groupby(['City', 'year']).label.value_counts(normalize=True).reset_index(name='label_ratio')
wohnflaeche = df.groupby(['City', 'year', 'label']).wohnflaeche.mean().reset_index(name='wohnflaeche_mean')
baujahr = df.groupby(['City', 'year', 'label']).baujahr.mean().reset_index(name='baujahr_mean')
rent_sqm = df.groupby(['City', 'year', 'label']).rent_sqm.mean().reset_index(name='rent_sqm_mean')
miete_mean = df.groupby(['City', 'year', 'label']).mietekalt.mean().reset_index(name='mietekalt_mean')
laufzeittage = df.groupby(['City', 'year', 'label']).laufzeittage.mean().reset_index(name='laufzeittage_mean')
anzahletagen = df.groupby(['City', 'year', 'label']).anzahletagen.mean().reset_index(name='anzahletagen_mean')
zimmeranzahl = df.groupby(['City', 'year', 'label']).zimmeranzahl.mean().reset_index(name='zimmeranzahl_mean')








df.groupby(['City', 'year', 'label']).ausstattung.value_counts().reset_index(name='ausstattung_count')
summary =miete_mean.merge(wohnflaeche, left_on=['City', 'year', 'label'], right_on=['City', 'year', 'label']) \
.merge(baujahr, left_on=['City', 'year', 'label'], right_on=['City', 'year', 'label']) \
.merge(rent_sqm, left_on=['City', 'year', 'label'], right_on=['City', 'year', 'label']) \
.merge(laufzeittage, left_on=['City', 'year', 'label'], right_on=['City', 'year', 'label']) \
.merge(anzahletagen, left_on=['City', 'year', 'label'], right_on=['City', 'year', 'label']) \
.merge(zimmeranzahl, left_on=['City', 'year', 'label'], right_on=['City', 'year', 'label']) \
.merge(city_count, left_on=['City', 'year', 'label'], right_on=['City', 'year', 'label'])
summary
plt.figure(figsize=(12,9))


sns.scatterplot(data=summary.query('City == "München" '), x='mietekalt_mean', y='wohnflaeche_mean', hue='year', palette='husl')
berlin = summary.query('City == "Berlin"')
berlin.groupby(['year', 'label']).sum()
label_ratio = df.groupby(['City', 'year']).label.value_counts(normalize=True).reset_index(name='label_ratio')
city_count = df.groupby(['City', 'year']).label.value_counts(normalize=True).reset_index(name='label_ratio')
city_count.groupby(['City', 'year']).sum()
berlin = label_ratio.query('City == "Berlin"')
plt.figure(figsize=(12,9))
sns.lineplot(data=label_ratio.query('City == "München"'), x='year', y='label_ratio', hue='label', palette='husl')
plt.legend(labels=['very high priced', 'high priced', 'normal priced', 'low priced', 'very low priced'])
summary.head()
plt.figure(figsize=(12, 9))
sns.barplot(data=summary, x='label', y='mietekalt_mean', \
            order=['very high priced', 'high priced', 'normal priced', 'low priced', 'very low priced'], color=blue);

plt.figure(figsize=(12, 9))
sns.barplot(data=summary, x='label', y='zimmeranzahl_mean', \
            order=['very high priced', 'high priced', 'normal priced', 'low priced', 'very low priced'], color=blue);


plt.figure(figsize=(12, 9))
sns.barplot(data=summary, x='label', y='wohnflaeche_mean', \
            order=['very high priced', 'high priced', 'normal priced', 'low priced', 'very low priced'], color=blue);


plt.figure(figsize=(12, 9))
sns.barplot(data=summary, x='label', y='anzahletagen_mean', \
            order=['very high priced', 'high priced', 'normal priced', 'low priced', 'very low priced'], color=blue);


df.to_pickle('data/rent_clustering_done.pkl')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18017006257019091, 'fraction_numerical': 0.009626183218353923, 'mean_word_length': 6.5380894800483675, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '153215', 'license': 'MIT', 'n_tokens_mistral': 2350, 'n_tokens_neox': 2233, 'n_words': 403, 'path': 'Clustering.ipynb', 'repo_name': 'DigiData-Flo/German_Hoisung_Prices', 'size': '6233'}