<jupyter_start><jupyter_text>Project: Identify Customer Segments<jupyter_code># import libraries here; add more as necessary
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# magic word for producing visualizations in notebook
%matplotlib inline
# Load in the general demographics data.
azdias = pd.read_csv('Udacity_AZDIAS_Subset.csv', sep=';')

# Load in the feature summary file.
feat_info = pd.read_csv('AZDIAS_Feature_Summary.csv', sep=';')
# Check the structure of the data after it's loaded (e.g. print the number of
# rows and columns, print the first few rows).

display(azdias.head())
display(f"Shape of demographic general: {azdias.shape}")
#display(azdias.info())
azdias.describe()
feat_info.head(8)<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>%%time

for index, row in feat_info.iterrows():
    attrib_name = row['attribute']
    data = azdias[attrib_name]

    if row['missing_or_unknown'] != '[]':
        miss_codes = row['missing_or_unknown'][1:-1].split(',')
    else:
        miss_codes = []
    if miss_codes:
        for code in miss_codes:
            # try-except block to capture 'X' and 'XX' codes
            try:
                data = data.replace([int(code)], np.nan)
            except:
                data = data.replace([code], np.nan)

    azdias[attrib_name] = data
# Perform an assessment of how much missing data there is in each column of the
# dataset.

# Percentage of missing values in each columns

plt.figure(figsize=(20,5))
(azdias.isnull().sum()/len(azdias)).plot.bar()
# Remove the outlier columns from the dataset. (You'll perform other data
# engineering tasks such as re-encoding and imputation later.)

to_drop = ['AGER_TYP', 'KK_KUNDENTYP', 'GEBURTSJAHR', 'TITEL_KZ', 'KBA05_BAUMAX', 'ALTER_HH']

azdias.drop(labels = to_drop, axis = 1, inplace = True)
# Without outliers
plt.figure(figsize=(20,5))
(azdias.isnull().sum()/len(azdias)).plot.bar()
# How much data is missing in each row of the dataset?

azdias['nb_missing_by_row'] = azdias.isnull().sum(axis=1)
plt.figure(figsize=(20,5))

row_nans = azdias.isna().sum(axis=1)

plt.hist(row_nans, bins = np.arange(-0.5,row_nans.max()+1,1))
plt.yticks(np.arange(0,600000+100000,100000),['0','100k','200k','300k','400k','500k','600k'])
plt.ylabel('# of data points')
plt.xlabel('# of missing values')
#19 is a good threshold
# Write code to divide the data into two subsets based on the number of missing
# values in each row.

azdias_missing = azdias[azdias['nb_missing_by_row'] > 19]
azdias_few_missing = azdias[azdias['nb_missing_by_row'] <= 19]
# Compare the distribution of values for at least five columns where there are
# no or few missing values, between the two subsets.

# ALTERSKATEGORIE_GROB: estimated age
# ANREDE_KZ: gender
# CJT_GESAMTTYP: customer journey typology
# FINANZTYP: most descriptive financial type for individual
# GFK_URLAUBERTYP: vacation habits

def plot_comparison(col_to_compare, title='No title'):
    plt.figure(figsize=(20,5))
    
    plt.subplot(1, 2, 1)
    plt.title(title+': missing data')
    sns.countplot(x = col_to_compare, data = azdias_missing,
                  order = azdias_missing[col_to_compare].value_counts().index)
    
    plt.subplot(1, 2, 2)
    plt.title(title+': no missing data')
    sns.countplot(x = col_to_compare, data = azdias_few_missing,
                  order = azdias_few_missing[col_to_compare].value_counts().index)
    
    plt.show()
plot_comparison('ALTERSKATEGORIE_GROB', 'Estimated age')
plot_comparison('ANREDE_KZ', 'Gender')
plot_comparison('CJT_GESAMTTYP', 'Customer journey typology')
plot_comparison('FINANZTYP', 'Financial type')
plot_comparison('GFK_URLAUBERTYP', 'Vacations habits')
print(f" Subset missing nb rows: {len(azdias_missing)}")
print(f" Subset few missing nb rows: {len(azdias_few_missing)}")<jupyter_output>Subset missing nb rows: 94144
 Subset few missing nb rows: 797077<jupyter_text>Between rows from the subset data with missing values and the orther one, there is clearly a difference when seeing the distribution of the 5 variables chosen.<jupyter_code># How many features are there of each data type?

feat_info.groupby('type')['attribute'].count()<jupyter_output><empty_output><jupyter_text>* HERE WE CONTINU THE ANALYSIS WITH FEW MISSING DATA DATASET : azdias_few_missing<jupyter_code># Assess categorical variables: which are binary, which are multi-level, and
# which one needs to be re-encoded?

l = list(feat_info[feat_info.type == 'categorical'].attribute)
l.remove('AGER_TYP')
l.remove('KK_KUNDENTYP')
l.remove('TITEL_KZ')

# Iterate over catecorical non numeric feature
for i in list(azdias_few_missing[l].select_dtypes(include='O')):
    if len(azdias_few_missing[i].value_counts()) == 2:
        azdias_few_missing = pd.get_dummies(azdias_few_missing, columns = [i])
    else:
        # these features talk about Life Stage and I already have this kind of information with other feature
        azdias_few_missing = azdias_few_missing.drop(labels=i, axis=1)
azdias_few_missing.shape<jupyter_output><empty_output><jupyter_text>* We have a lot of variables with 3 or more distincts values, that why I decided not investigate these columns and just encode them because I don't know if they are important. It's more interesting to have a few number of columns but I can reduce this number by applying PCA or TSNE after. Knowing these dimentionality reduction method, that why I decided to keep these column.<jupyter_code># Investigate "PRAEGENDE_JUGENDJAHRE" and engineer two new variables.

# 1,2 : 40s
# 3,4 : 50s
# 5,6,7 : 60s
# 8,9 : 70s
# 10-15 : 80s


# 1 3 5 8 10 12 14 --> mainstream
# 2 4 6 7 9 11 13 15 --> avantgarde

def investigate_preagende(data):
    if data in (1, 2):
        return '40s'
    elif data in (3, 4):
        return '50s'
    elif data in (5, 6, 7):
        return '60s'
    elif data in (8, 9):
        return '70s'
    else:
        return '80s'
    
    
azdias_few_missing['decade'] = azdias_few_missing.PRAEGENDE_JUGENDJAHRE.apply(investigate_preagende)

def investigate_preagende2(data):
    if data in (1, 3, 5, 8, 10, 12, 14):
        return 'mainstream'
    else:
        return 'avantgarde'
    
azdias_few_missing['movement'] = azdias_few_missing.PRAEGENDE_JUGENDJAHRE.apply(investigate_preagende2)

azdias_few_missing = pd.get_dummies(azdias_few_missing, columns = ['decade', 'movement'])

azdias_few_missing = azdias_few_missing.drop(labels = 'PRAEGENDE_JUGENDJAHRE', axis = 1)


# Investigate "CAMEO_INTL_2015" and engineer two new variables.

azdias_few_missing.CAMEO_INTL_2015 = azdias_few_missing.CAMEO_INTL_2015.astype('float64')

# Tens
azdias_few_missing['CAMEO_INTL_WEALTH'] = azdias_few_missing.CAMEO_INTL_2015.apply(lambda x: divmod(x, 10)[0])

# ones
azdias_few_missing['CAMEO_INTL_LIFESTAGE'] = azdias_few_missing.CAMEO_INTL_2015.apply(lambda x: divmod(x, 10)[1])
    
azdias_few_missing = azdias_few_missing.drop(labels = 'CAMEO_INTL_2015', axis = 1)

azdias_few_missing['CAMEO_INTL_WEALTH'] = azdias_few_missing['CAMEO_INTL_WEALTH'].astype('category', ordered = True)

azdias_few_missing['CAMEO_INTL_WEALTH'] = azdias_few_missing['CAMEO_INTL_WEALTH'].cat.codes
# If there are other re-engineering tasks you need to perform, make sure you
# take care of them here. (Dealing with missing data will come in step 2.1.)



# The other mixed values are either about life stage, age or wealth wich are correlated with other feature in the dataset,
# so I decide to drop them

to_drop = ['LP_LEBENSPHASE_FEIN', 'LP_LEBENSPHASE_GROB', 'WOHNLAGE', 'PLZ8_BAUMAX']

azdias_few_missing = azdias_few_missing.drop(labels = to_drop, axis = 1)
# Do whatever you need to in order to ensure that the dataframe only contains
# the columns that should be passed to the algorithm functions.
azdias_few_missing = azdias_few_missing.drop(labels = 'nb_missing_by_row', axis = 1)
def clean_data(df):
    """
    Perform feature trimming, re-encoding, and engineering for demographics
    data
    
    INPUT: Demographics DataFrame
    OUTPUT: Trimmed and cleaned demographics DataFrame
    """
    for index, row in feat_info.iterrows():
        attrib_name = row['attribute']
        data = df[attrib_name]

        if row['missing_or_unknown'] != '[]':
            miss_codes = row['missing_or_unknown'][1:-1].split(',')
        else:
            miss_codes = []
        if miss_codes:
            for code in miss_codes:
                # try-except block to capture 'X' and 'XX' codes
                try:
                    data = data.replace([int(code)], np.nan)
                except:
                    data = data.replace([code], np.nan)

        df[attrib_name] = data
        
    to_drop = ['AGER_TYP', 'KK_KUNDENTYP', 'GEBURTSJAHR', 'TITEL_KZ', 'KBA05_BAUMAX', 'ALTER_HH']
    df.drop(labels = to_drop, axis = 1, inplace = True)
        
    df['nb_missing_by_row'] = df.isnull().sum(axis=1)
    df = df[df['nb_missing_by_row'] <= 19]  
    
    l = list(feat_info[feat_info.type == 'categorical'].attribute)
    l.remove('AGER_TYP')
    l.remove('KK_KUNDENTYP')
    l.remove('TITEL_KZ')

    # Iterate over catecorical non numeric feature
    for i in list(df[l].select_dtypes(include='O')):
        if len(df[i].value_counts()) == 2:
            df = pd.get_dummies(df, columns = [i])
        else:
            # these features talk about Life Stage and I already have this kind of information with other feature
            df.drop(labels=i, axis=1, inplace = True)
            
            
    def investigate_preagende(data):
        if data in (1, 2):
            return '40s'
        elif data in (3, 4):
            return '50s'
        elif data in (5, 6, 7):
            return '60s'
        elif data in (8, 9):
            return '70s'
        else:
            return '80s'

    df['decade'] = df.PRAEGENDE_JUGENDJAHRE.apply(investigate_preagende)

    def investigate_preagende2(data):
        if data in (1, 3, 5, 8, 10, 12, 14):
            return 'mainstream'
        else:
            return 'avantgarde'

    df['movement'] = df.PRAEGENDE_JUGENDJAHRE.apply(investigate_preagende2)
    df = pd.get_dummies(df, columns = ['decade', 'movement'])
    df.drop(labels = 'PRAEGENDE_JUGENDJAHRE', axis = 1, inplace = True)


    # Investigate "CAMEO_INTL_2015" and engineer two new variables.
    df.CAMEO_INTL_2015 = df.CAMEO_INTL_2015.astype('float64')

    # Tens
    df['CAMEO_INTL_WEALTH'] = df.CAMEO_INTL_2015.apply(lambda x: divmod(x, 10)[0])

    # ones
    df['CAMEO_INTL_LIFESTAGE'] = df.CAMEO_INTL_2015.apply(lambda x: divmod(x, 10)[1])
    df.drop(labels = 'CAMEO_INTL_2015', axis = 1, inplace = True)
    df['CAMEO_INTL_WEALTH'] = df['CAMEO_INTL_WEALTH'].astype('category', ordered = True).cat.codes    
    
    to_drop = ['LP_LEBENSPHASE_FEIN', 'LP_LEBENSPHASE_GROB', 'WOHNLAGE', 'PLZ8_BAUMAX']
    df.drop(labels = to_drop, axis = 1, inplace = True)    
    df.drop(labels = 'nb_missing_by_row', axis = 1, inplace = True)
    
    
    return df<jupyter_output><empty_output><jupyter_text>Feature Transformation<jupyter_code># If you've not yet cleaned the dataset of all NaN values, then investigate and
# do that now.
from sklearn.impute import SimpleImputer

imp = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
X = imp.fit_transform(azdias_few_missing)
# Apply feature scaling to the general population demographics data.
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
X = scaler.fit_transform(X)
# Apply PCA to the data.

from sklearn.decomposition import PCA

pca = PCA()
pca.fit(X)
plt.figure(figsize=(20,10))

#Plotting the Cumulative Summation of the Explained Variance
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.axhline(y=0.95, color='r', linestyle='-')
plt.axvline(x=51, color='r', linestyle='-')
plt.xlabel('Number of Components')
plt.ylabel('Variance (%)') #for each component
plt.show()
# Re-apply PCA to the data while selecting for number of components to retain.

pca = PCA(n_components = 51)
pca.fit(X)
transformed = pca.transform(X)<jupyter_output><empty_output><jupyter_text>We can see thanks to the plot that 51 component can explain 95% of the data so, I decided to choose this number as my intrinsic dimention.<jupyter_code># Map weights for the first principal component to corresponding feature names
# and then print the linked values, sorted by weight.
# HINT: Try defining a function here or in a new cell that you can reuse in the
# other cells.
dataFrame_scaled = pd.DataFrame(X,columns = azdias_few_missing.columns).head()

def map_pcweight_feature_name(id_component = 1):

    s = pd.DataFrame(pca.components_,
                         columns=dataFrame_scaled.columns).iloc[id_component-1]

    s2 = pd.DataFrame(dict(s), index=['PC'+str(id_component)])
    del s
    s2 = s2.T

    s2['feature_name'] = s2.index
    s2 = (
        s2[['feature_name', 'PC'+str(id_component)]]
        .reset_index(drop=True)
        .sort_values(by='PC'+str(id_component))
    )
    
    return s2

display(map_pcweight_feature_name(1).head())
display(map_pcweight_feature_name(1).tail())
# Map weights for the second principal component to corresponding feature names
# and then print the linked values, sorted by weight.

display(map_pcweight_feature_name(2).head())
display(map_pcweight_feature_name(2).tail())
# Map weights for the third principal component to corresponding feature names
# and then print the linked values, sorted by weight.

display(map_pcweight_feature_name(3).head())
display(map_pcweight_feature_name(3).tail())<jupyter_output><empty_output><jupyter_text>Clustering<jupyter_code>from sklearn.cluster import KMeans

cluster_range = range(2, 20)
inertias = []

for k in cluster_range:
    model = KMeans(n_clusters = k)
    model.fit(transformed)
    inertias.append(model.inertia_)
#plt.figure(figsize=(15, 5))
plt.plot(range(2, 20), inertias, '-o')
plt.xlabel('nb of cluster')
plt.ylabel('inertia')
# Re-fit the k-means model with the selected number of clusters and obtain
# cluster predictions for the general population demographics data.

kmeans = KMeans(n_clusters=14).fit(transformed)
pop_result = kmeans.predict(transformed)
print(f"Population: {len(pop_result)}")
print(f"Nb Clusters: {len(kmeans.cluster_centers_)}")
x = [i for i in range(14)]
pop_list = []
for i in range(14):
    pop_list.append(np.sum(pop_result==i)/len(pop_result))

df = pd.DataFrame({'cluster' : x, 'population' : pop_list})
df.plot(x = 'cluster', y = 'population', kind='bar', figsize=(20,6), title = 'Percentage of poeple in cluster (population)')
# Load in the customer demographics data.
customers = pd.read_csv('Udacity_CUSTOMERS_Subset.csv', sep=';')
%%time
# Apply preprocessing, feature transformation, and clustering from the general
# demographics onto the customer data, obtaining cluster predictions for the
# customer demographics data.

customers = clean_data(customers)
print(azdias_few_missing.shape)
print(customers.shape)
imp_customer = imp.transform(customers)
scaled_customer = scaler.transform(imp_customer)
transformed_customer = pca.transform(scaled_customer)
cust_result = kmeans.predict(transformed_customer)
# Compare the proportion of data in each cluster for the customer data to the
# proportion of data in each cluster for the general population.

population_list = []
customers_list = []
x = [i for i in range(14)]
for i in range(14):
    population_list.append(np.sum((pop_result == i))/len(pop_result))
    customers_list.append(np.sum((cust_result == i))/len(cust_result))


df = pd.DataFrame({'cluster' : x, 'population' : population_list, 'customers':customers_list})

#ax = sns.countplot(x='index', y = df_general['prop_1', 'prop_2'], data=df_general )
df.plot(x='cluster', y = ['population', 'customers'], kind='bar', figsize=(20,6))
plt.ylabel('percentage of persons in each cluster')
plt.show()
def relevant_feat_per_cluster(id_cluster):
    center_pca_inv = pca.inverse_transform(kmeans.cluster_centers_[id_cluster])
    df = pd.DataFrame(dict(zip(azdias_few_missing.columns, center_pca_inv)), index = ['center']).T.sort_values(by=['center'])
    return display(df.head()), display(df.tail())
relevant_feat_per_cluster(0)
relevant_feat_per_cluster(1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1156470949892407, 'fraction_numerical': 0.020965262834306792, 'mean_word_length': 4.352418558736426, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '31434', 'license': 'CC0-1.0', 'n_tokens_mistral': 5991, 'n_tokens_neox': 5565, 'n_words': 1433, 'path': 'Identify_Customer_Segments.ipynb', 'repo_name': 'Mdhvince/Customer_Segmentation', 'size': '16265'}