<jupyter_start><jupyter_text>Population Segmentation Dataset Objective - An unsupervised learning model to seek clusters in the Population Segmentation Data and US Census Data that reveal some sort of demographic traits the define their similarities for different regions (Indexed by concatenating State-County) in the United States 1. DATA COLLECTION WITH AWS S3- Use AWS S3 bucket is a storage location to Pop Segmentation dataset 2. EXPLORATORY DATA ANALYSIS & DATA CLEANING- Find missing values (replace them with mean/media/mode, KNN, or KMEANs) - plot distribution - plot pie chart  3. FEATURE ENGINEERING- KMeans and DBSCAN uses euclidean distance- We want to minimize the distance length between two points in Euclidean space (it's required to scale the data)- Find different ways to scale the data (Gussian Distribution)- Logarithmic Transformation- Box-Cox Transformation- Standard Scaler (z-score) 4. FEATURE SELECTION- Dropping low variance variables - Pearson Correlation - Principal Component Analysis<jupyter_code>import pandas as pd 
import numpy as np 
import boto3
import io
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.feature_selection import VarianceThreshold
import scipy.stats as stats
from pprint import pprint
import pylab<jupyter_output><empty_output><jupyter_text>1. DATA COLLECTION WITH AWS S3<jupyter_code># use boto3 client to get S3 bucket
s3_client = boto3.client('s3')
obj_list=s3_client.list_objects(Bucket='aws-ml-blog-sagemaker-census-segmentation')

# print objects in S3 bucket
files=[]
for contents in obj_list['Contents']:
    files.append(contents['Key'])
file_name=files[0]


data_object = s3_client.get_object(Bucket='aws-ml-blog-sagemaker-census-segmentation', Key=file_name)
data_body = data_object["Body"].read()
data_stream = io.BytesIO(data_body)

# create a pandas dataframe
df = pd.read_csv(data_stream, header=0, delimiter=",") 
df.head()
# Combine 'State' and 'County' and index the column
df.index = df['State'] + '-' + df['County']
# drop the 'CensusId', 'State', 'County' columns
df = df.drop(['CensusId', 'State', 'County'], axis=1)<jupyter_output><empty_output><jupyter_text>EXPLORATORY DATA ANALYSIS & DATA CLEANING<jupyter_code># provides a quantitative summary for each column (count, mean, std, min, max, etc)
df.describe()
print(f'Number of rows of DataFrame {df.shape[0]}')
print(f'Number of columns of DataFrame {df.shape[1]}')
# get datatype for each feature (string, float, int, bool, object, etc)
# State and County are object
pprint(df.columns.to_series().groupby(df.dtypes).groups)
features = df.columns
numerical_feat = [features for features in df.columns if df[features].dtypes != 'O']
print(f'Number of Numerical Features: {len(numerical_feat)}')

categorical_feat = [features for features in df.columns if df[features].dtypes == 'O']
print(f'Number of Categorical Features: {len(categorical_feat)}')<jupyter_output>Number of Numerical Features: 34
Number of Categorical Features: 0<jupyter_text>Find Missing Values<jupyter_code># find null values in DataFrame
if df.isnull().sum().any() == False:
    print('Data is Clean, No Null values found')
else:
    print('Found Null Values')
# create a pandas dataframe with data_type, num_null_values, and percentage of null_values for each feature
df_details = pd.DataFrame(df.dtypes).T.rename(index={0:'datatype'})
df_details = df_details.append(pd.DataFrame(df.isnull().sum()).T.rename(index={0:'num_null_values'}))
df_details = df_details.append(pd.DataFrame(df.isnull().sum()/len(df)*100).T.rename(index={0:'pct_null_values'}))
df_details<jupyter_output><empty_output><jupyter_text>Fill Null Values with Median since there's outliers<jupyter_code># Null values can be dropped, replaced by Median, Mean, or t-SNE
# Before filling Null values, analyz the data distribution for [ChildPoverty, IncomeErr, Income]
null_col = ['ChildPoverty', 'IncomeErr', 'Income']
plt.subplots(figsize=(15,8))
sns.boxplot(data=df[null_col], orient="h", palette="Set2")
# given the boxplot, there's alot of extreme values in IncomeErr and Income
# fill in null values with median, not mean (mean is sensitive to extreme values)
df = df.fillna(df.median())
# Verify if DataFrame still has null values 
if df.isnull().sum().any() == False:
    print('Data is Clean, No Null values found')
else:
    print('Found Null Values')
df.columns
# income_list features historgram 
income_list = ['Income', 'IncomeErr', 'IncomePerCap', 'IncomePerCapErr', 'Poverty', 'ChildPoverty']
plt.subplots(figsize=(15,8))
sns.histplot(df[income_list].astype(float))


# pie chart of income sum
sizes = [df['Income'].sum(),
         df['IncomeErr'].sum(),
         df['IncomePerCap'].sum(),
         df['IncomePerCapErr'].sum(),
         df['Poverty'].sum(),
         df['ChildPoverty'].sum()
         
        ]
fig1, ax1 = plt.subplots(figsize=(8,8))
ax1.pie(sizes, labels=income_list, autopct='%1.2f%%')
plt.show()

# occupation_list features historgram 
occupation_list = ['Professional', 'Service', 'Office', 'Construction', 'Production']
plt.subplots(figsize=(15,8))
sns.histplot(df[occupation_list].astype(float))

sizes = [df['Professional'].sum(),
         df['Service'].sum(),
         df['Office'].sum(),
         df['Construction'].sum(),
         df['Production'].sum(),         
        ]
fig1, ax1 = plt.subplots(figsize=(8,8))
ax1.pie(sizes, labels=occupation_list, autopct='%1.2f%%', shadow=False)
plt.show()
# transportation features historgram 
transportation_list = ['Drive', 'Carpool', 'Transit', 'Walk', 'OtherTransp']
plt.subplots(figsize=(15,8))
sns.histplot(df[transportation_list].astype(float))


# pie chart of transportation_list
sizes = [df['Drive'].sum(),
         df['Carpool'].sum(),
         df['Transit'].sum(),
         df['Walk'].sum(),
         df['OtherTransp'].sum(),         
        ]
fig1, ax1 = plt.subplots(figsize=(8,8))
ax1.pie(sizes, labels=transportation_list, autopct='%1.2f%%', shadow=False)
plt.show()
# race features historgram  
race_list = ['Hispanic', 'White', 'Black', 'Native', 'Asian','Pacific']
plt.subplots(figsize=(15,8))
sns.histplot(df[race_list].astype(float))

# pie chart of race_list
sizes = [df['Hispanic'].sum(),
         df['White'].sum(),
         df['Black'].sum(),
         df['Native'].sum(),
         df['Asian'].sum(),
         df['Pacific'].sum()
         
        ]
fig1, ax1 = plt.subplots(figsize=(8,8))
ax1.pie(sizes, labels=race_list, autopct='%1.2f%%', shadow=False)
plt.show()<jupyter_output><empty_output><jupyter_text>VISUALIZE PEARSON CORRELATION  Helps identify any strong correlation between variables<jupyter_code># compute corr array and generate a mask for the upper triangle
corr = df.corr()
mask = np.triu(np.ones_like(corr, dtype=bool))
plt.subplots(figsize=(15,8))

# Generate a custom diverging colormap
cmap = sns.diverging_palette(230, 20, as_cmap=True)

# plt heatmap with mask and ratio
sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})

# looks like [Poverty, ChildPoverty] are highly correlated with [Income, Income Cap]<jupyter_output><empty_output><jupyter_text>FEATURE ENGINEERING   Define Index as State + County<jupyter_code>old_columns = df.columns<jupyter_output><empty_output><jupyter_text>Box-Plot Identifies outliers<jupyter_code>gender_list = ['Men', 'Women',]
plt.subplots(figsize=(20,7))
sns.boxplot(data=df[gender_list], orient="h", palette="Set2")<jupyter_output><empty_output><jupyter_text>Box Plot for Race Features<jupyter_code>plt.subplots(figsize=(20,7))
sns.boxplot(data=df[race_list], orient="h", palette="Set2")<jupyter_output><empty_output><jupyter_text>Box Plot for Occupations Features<jupyter_code>plt.subplots(figsize=(20,7))
sns.boxplot(data=df[occupation_list], orient="h", palette="Set2")<jupyter_output><empty_output><jupyter_text>Box Plot for Transportation Features<jupyter_code>plt.subplots(figsize=(20,7))
sns.boxplot(data=df[transportation_list], orient="h", palette="Set2")<jupyter_output><empty_output><jupyter_text>Box Plot for Income Features<jupyter_code>plt.subplots(figsize=(20,7))
sns.boxplot(data=df[income_list], orient="h", palette="Set2")<jupyter_output><empty_output><jupyter_text>Implement Guassian Transformation for each cluster- Logarithmic Transformation- Reciprocal Transformation- Square-Root Transformation- Exponential Transformation- Box-Cox Transformation<jupyter_code># plot two diagram (distribution histogram and boxplot)
def plot_distribution(df, feature):
	plt.figure(figsize=(10,6))
	# first row, 1st column
	plt.subplot(1, 2, 1)
	df[feature].hist()
	# first row, 2 column
	plt.subplot(1, 2, 2)
	stats.probplot(df[feature], dist='norm', plot=pylab)
	plt.show()
# plot two diagram (distribution histogram and boxplot)
def plot_data_transformation(df, feature, feature_transform):
	plt.figure(figsize=(20,6))
	plt.subplot(1, 4, 1)
	stats.probplot(df[feature], dist='norm', plot=pylab)

	plt.subplot(1, 4, 2) 
	df[feature].hist()
     
	plt.subplot(1, 4, 3)
	stats.probplot(df[feature_transform], dist='norm', plot=pylab)

	plt.subplot(1, 4, 4)
	df[feature_transform].hist()
	plt.show()
# Gender Transformation using Logarithmic 
def transform_feature_log(df, feature):
    df[feature+'_log'] = np.log(df[feature])

for i in gender_list:
        transform_feature_log(df, i)

plot_data_transformation(df, 'Women', 'Women_log')
def transform_feature_scale(df, feature):
    scaler = StandardScaler()
    df[[feature+'_scale']] = scaler.fit_transform(df[[feature]])
    
for i in race_list:
    transform_feature_scale(df, i)
# Occupation Tranformation 
def transform_feature_box(df, feature):
    df[feature+'_boxcox'], _ = stats.boxcox(df[feature] + 1)
    
for i in occupation_list:
    transform_feature_log(df, i)
    
plot_data_transformation(df, 'Professional', 'Professional_log')
# Transportation Transformation using Boxcox 
for i in transportation_list:
    transform_feature_box(df, i)

plot_data_transformation(df, 'Walk', 'Walk_boxcox')
# Income Transformation using Logarithmic 
for i in income_list:
        transform_feature_log(df, i)

plot_data_transformation(df, 'IncomePerCapErr', 'IncomePerCapErr_log')
transform_feature_log(df, 'TotalPop')
plot_data_transformation(df, 'TotalPop', 'TotalPop_log')
scaler = StandardScaler()
df[["MeanCommute_scale"]] = scaler.fit_transform(df[["MeanCommute"]])

plot_distribution(df, 'MeanCommute_scale')
df = df.drop(old_columns, axis = 1)<jupyter_output><empty_output><jupyter_text>Feature Selection- Dropping low variance variables - Pearson Correlation - Principal Component Analysis<jupyter_code>def clean_dataset(df):
    assert isinstance(df, pd.DataFrame), "df needs to be a pd.DataFrame"
    df.dropna(inplace=True)
    indices_to_keep = ~df.isin([np.nan, np.inf, -np.inf]).any(1)
    return df[indices_to_keep].astype(np.float64)

df = clean_dataset(df)
# compute corr array and generate a mask for the upper triangle
corr = df.corr()
mask = np.triu(np.ones_like(corr, dtype=bool))
plt.subplots(figsize=(15,8))

# Generate a custom diverging colormap
cmap = sns.diverging_palette(230, 20, as_cmap=True)

# plt heatmap with mask and ratio
sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})
var_threshold = VarianceThreshold(threshold=0.05)
var_threshold.fit(df)

# get the number of feature coumns with 0 variance 
constant_col = [col for col in df.columns if col not in df.columns[var_threshold.get_support()]]
# drop feature with 0 variance (constanct features) 
df.drop(constant_col, axis=1, inplace=True)

print(constant_col)
def drop_corr(df, threshold):
	# set of all the names of correlated columns
    col_corr = set()
    corr_matrix = df.corr()
    for i in range(len(corr_matrix.columns)):
        for j in range(i):
			# get the coefficient value
            if corr_matrix.iloc[i, j] > threshold:
                colname = corr_matrix.columns[i]
                col_corr.add(colname)
    return col_corr
    
corr_features = drop_corr(df, 0.80)
print(corr_features)
df.drop(corr_features, axis=1, inplace=True)

# compute corr array and generate a mask for the upper triangle
corr = df.corr()
mask = np.triu(np.ones_like(corr, dtype=bool))
plt.subplots(figsize=(15,8))

# Generate a custom diverging colormap
cmap = sns.diverging_palette(230, 20, as_cmap=True)

# plt heatmap with mask and ratio
sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})
df.to_csv('../inputs/population_seg_clean.csv', index=False)
# implement this in another rep py file 
# PCA for dimensionality reduction
# Clustering transformed data with k-means
# Use Kmeans clustering model and evalute model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1314414906057535, 'fraction_numerical': 0.013409214937241756, 'mean_word_length': 5.6397515527950315, 'pattern_counts': {'":': 3, '<': 56, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '511817', 'license': 'MIT', 'n_tokens_mistral': 4564, 'n_tokens_neox': 4296, 'n_words': 1055, 'path': 'notebook/population_segmentation.ipynb', 'repo_name': 'jf20541/KmeansDbscanPCA', 'size': '12827'}