<jupyter_start><jupyter_text>K-Means Machine Learning AlgorithmThis notebook uses the K Means algorithm to create personas based on person and household attributes from PUMs data. This is an unsupervised learning technique because there are no pre-labeling of the personas. The attributes that are choosen will work with clustering and are easy to communicate. Then we will be able to use the data to describe meaningful attributes of a real type of transportation users.<jupyter_code>import math
import numpy as np
import pandas as pd
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from sklearn.decomposition import PCA
# Loading the preprocessed data 
df_data = pd.read_csv('pums_processed.csv', index_col = 0)
# Droping some characteristics for a better clustering 
df_data = df_data.drop(['AGEP', 'WHITE_ALONE', 'BLACK_ALONE', 'NATIVE_INDIAN', 'ASIAN_ALONE', 'PACIFIC_ISLANDER',
             'OTHER_RACE', 'HISP', 'DOWNTOWN', 'NORTHEAST', 'NORTHWEST', 'SOUTHEAST', 'WEST'], axis=1)
# Preparing for K-means clustering --> dropping all NaNs and normalizing the data 
df_nonan = df_data.dropna()
index_nomissing = df_nonan['SERIALNO']
df_nonan = df_nonan.set_index('SERIALNO')
normalized_cluster = (df_nonan-df_nonan.mean())/df_nonan.std()
normalized_cluster.head(5)
# K-Mean clustering where n_cluster=5 
CLUSTER = 'cluster'
kmeans = KMeans(n_clusters=5, random_state=0).fit(normalized_cluster)
clustered = pd.DataFrame({CLUSTER: kmeans.labels_}, index= index_nomissing)

df_final = df_nonan.join(clustered)
#df_final.head(-5)
# Checking how many data points are in each cluster 
for i in range(len(df_final['cluster'].unique())):
    print(len(df_final[df_final['cluster'] == i]))
# Aggregating to the cluster level 
cluster_summary = df_final.groupby(['cluster'], as_index = True).agg({'DDRS':'mean', 'ENG':'mean', 'DEAR':'mean', 'DEYE':'mean', 'DOUT':'mean',
                                                            'MIG':'mean', 'JWMNP':'mean', 'BIKING':'mean', 'DRIVING':'mean', 
                                                            'TRANSIT':'mean', 'WALKING':'mean', 'MALE':'mean', 'FEMALE':'mean', 'SCHL':'mean'})

#cluster_summary.head(5)
d = {'cluster': [1, 2, 3, 4, 5]}
df = pd.DataFrame(data=d)
#cluster_summary = pd.concat([cluster_summary, df], axis=1)
cluster_summary.head(5)<jupyter_output><empty_output><jupyter_text>Calculating Weights into each Summary Cluster<jupyter_code>def multiply_weights(column, weights):
    return column * weights

def divide_weights(column, weights):
    return column / weights

def cluster_summary_with_weights(clustered_dat):
    
    input_dat = clustered_dat[['ENG', 'DEYE', 'DOUT', 'MIG', 'JWMNP', 
                        'BIKING', 'DRIVING', 'TRANSIT',
                        'WALKING', 'MALE', 'FEMALE', 'SCHL', 'WGTP']].copy()
    
    # multiply weights to each column of dat
    weighted_dat = input_dat.apply(lambda x: multiply_weights(x, normalized_cluster['WGTP']))
    weighted_dat = pd.concat([weighted_dat, normalized_cluster['WGTP'], cluster_summary['cluster']], axis = 1)
    
    cluster_sums = weighted_dat.groupby(['cluster'], as_index = True).agg('sum')
    
    weighted_cluster_means = cluster_sums.apply(lambda x: divide_weights(x, cluster_sums['WGTP']))
    # drop WEIGHT as all value will be 1.0
    weighted_cluster_means = weighted_cluster_means.drop(columns = 'WGTP')
    
    
    # add a neutral element (average/baseline for the personas)
    weighted_neutral = pd.Series(weighted_dat.sum())
    weighted_neutral = weighted_neutral / weighted_neutral['WGTP']
    # drop cluster/weight (used in calculation but unnecessary for further analysis) 
    weighted_neutral = weighted_neutral.drop(['cluster', 'WGTP'])

    weighted_cluster_means = weighted_cluster_means.append(weighted_neutral, ignore_index=True)
        
    weighted_cluster_means = weighted_cluster_means.set_index(weighted_cluster_mean)

    return(weighted_cluster_means)
#weighted_cluster_summary = cluster_summary_with_weights(df_final)
#weighted_cluster_summary<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1186399217221135, 'fraction_numerical': 0.004403131115459882, 'mean_word_length': 4.64, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '296674', 'license': 'MIT', 'n_tokens_mistral': 1376, 'n_tokens_neox': 1261, 'n_words': 315, 'path': 'notebooks/KMeansModel.ipynb', 'repo_name': 'monolyst/Seattle-Mobility-Index', 'size': '4088'}