<jupyter_start><jupyter_text>Import Libraries<jupyter_code># import libraries here; add more as necessary
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans, MiniBatchKMeans
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_auc_score
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
# magic word for producing visualizations in notebook
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Data Visualization<jupyter_code>azdias = pd.read_csv('../capstone_data/Udacity_AZDIAS_052018.csv',low_memory=False)
azdias.head()
azdias.describe()
#Plotting the proportion of null values top 15 columns
(azdias.isnull().sum().sort_values(ascending=False)/azdias.shape[0])[:15].plot(kind='bar')<jupyter_output><empty_output><jupyter_text>These values are not correct. Because, in the 'DIAS Attributes - Values 2017.xlsx' file, for columns certain values are unknown. For example, in the column AGER_TYP value -1 is said to be unknown. Similarly other columns have some values which are need to be mapped to NAN.<jupyter_code>#Categorical columns
azdias.select_dtypes(include='object').head()<jupyter_output><empty_output><jupyter_text>These categorical features have to be converted into numerical features. The columns 'CAMEO_DEUG_2015' and 'CAMEO_INTL_2015' are actually numerical but there some values in the columns which are 'XX' or 'X' that are to be replaced by NAN. The columns 'D19_LETZTER_KAUF_BRANCHE','EINGEFUEGT_AM','OST_WEST_KZ' can be dropped because they are too informative.  Data Preprcocessing<jupyter_code>def Attribute_Unknown_Dict(path,df):
    '''
    Purpose: Map all the attributes to their unknown values in a dictionary
    
    Input: PATH TO DIAS Attributes - Values 2017.xlsx file, in str format
    
    Output:
    new_dict:Dictionary with attributes as key and the unknown meaning values in a list
    '''
    #reading the xlsx file and storing it in a dataframe
    attributes=pd.read_excel(path,skiprows=1)
    
    #formatting
    attributes=attributes.drop(['Unnamed: 0'],axis=1,inplace=False)
    
    #Droping all the rows with nan values. Since, all the first values refer to the unknown meaning category\
    #Only keeping the first row for each attribute works
    attributes=attributes.dropna()
    
    new_dict={}
    for i in range(attributes.shape[0]):
        #checking if it corresponds to the unknown value
        if ('unknown' in attributes['Meaning'].iloc[i].split()) or(attributes['Meaning'].iloc[i] == 'no transaction known'):
            new_list=[]
            if type(attributes['Value'].iloc[i])==int:
                new_list.append(attributes['Value'].iloc[i])
            else:
                for j in attributes['Value'].iloc[i].split(','):
                    new_list.append(int(j))

            new_dict[attributes['Attribute'].iloc[i]]=new_list
    
    #These columns are not present in  the actual dataset
    new_dict.pop('BIP_FLAG')
    
    new_dict.pop('GEOSCORE_KLS7')
    new_dict.pop('HAUSHALTSSTRUKTUR')
    new_dict.pop('SOHO_FLAG')
    new_dict.pop('WACHSTUMSGEBIET_NB')
    
    
    #Removing name '_RZ' from the end of each column name
    for i in new_dict:
        if i not in df.columns:
            new_dict[i[:-3]]=new_dict.pop(i)
    
    #removing the last key_value pair
    new_dict.pop('')
    
    
    return new_dict
attribute_dict=Attribute_Unknown_Dict('../capstone_data/DIAS Attributes - Values 2017.xlsx',azdias)
attribute_dict<jupyter_output><empty_output><jupyter_text>Mapping the unknown values to nan with the help of the attribute_dict<jupyter_code>#replacing unknown values with nan
def Map_unknown_to_NAN(df,attribute_dict):
    '''
    Replace the unknown values with NAN values in the df
    ARGS:
    df: Dataframe on which the mapping takes place like azdias
    attribute_dict: Dict with attribute as keys and values which are to replaced with NAN
    
    Output:
    df: transformed df with more null values
    
    '''
    #Replacing 
    for key,val in attribute_dict.items():
        for j in val:
            df[key]=df[key].replace(j,np.nan)          
    return df
azdias=Map_unknown_to_NAN(azdias,attribute_dict)
(azdias.isnull().sum().sort_values(ascending=False)/azdias.shape[0])[:15].plot(kind='bar')
plt.title('Proportion of null values')
plt.xlabel('features')
plt.ylabel('% of null values')<jupyter_output><empty_output><jupyter_text>These columns can be dropped because more than 50% of the columns are NAN. We need to store these column names to apply the same concept to the customers dataframe.<jupyter_code>#listing the columns which have more than 50% null values
drop_col_list=list(azdias.isnull().sum(axis=0)[azdias.isnull().sum(axis=0)>0.50*(azdias.shape[0])]\
               .reset_index()['index'])
print('Len :',len(drop_col_list))
print(drop_col_list)
azdias.drop(drop_col_list,axis=1,inplace=True)
#shape of the azdias df
#42 features were removed
azdias.shape
#Certain values of CAMEO_INTL_2015 and CAMEO_DEUG_2015' are as follows:
azdias['CAMEO_INTL_2015'][azdias['CAMEO_INTL_2015']=='XX']<jupyter_output><empty_output><jupyter_text>We need to replace these values with NAN value and convert CAMEO_INTL_2015 and CAMEO_DEUG_2015 into numerical entries<jupyter_code>def cat_to_num(df):
    '''
    converting columns which are categorical to numerical and droping other categorical columns
    INput: 
    df: DataFrame to be processed
    
    Output:
    df: After droping and converting categorical columns
    
    '''
    
    #converting CAMEO_INTL_2015 AND CAMEO_DEUB_2015 into numerical values
    intl=[]
    deug=[]
    for i in range(len(df['CAMEO_INTL_2015'])):
        if type(df['CAMEO_INTL_2015'].iloc[i])==str and df['CAMEO_INTL_2015'].iloc[i][0]=='X':
            intl.append(np.nan)
        else:
            intl.append(float(df['CAMEO_INTL_2015'].iloc[i]))
        if type(df['CAMEO_INTL_2015'].iloc[i])==str and df['CAMEO_DEUG_2015'].iloc[i][0]=='X':
            deug.append(np.nan)
        else:
            deug.append(float(df['CAMEO_DEUG_2015'].iloc[i]))
    
    #droping the original columns
    df=df.drop(['CAMEO_INTL_2015','CAMEO_DEUG_2015'],axis=1,inplace=False)
    #Adding new columns
    df['CAMEO_INTL_2015']=intl
    df['CAMEO_DEUG_2015']=deug
    
    #droping 'LNR' AND 'VERDICHTUNGSRAUM' columns
    #droping the 'LP_FAMILIE_GROB' column because it very similar 'LP_FAMILIE_FEIN'

    df=df.drop(['LNR','VERDICHTUNGSRAUM','LP_FAMILIE_GROB'],axis=1,inplace=False)
    
    return df
    
azdias=cat_to_num(azdias)<jupyter_output><empty_output><jupyter_text>Mixed Categories can be segregated into individual categories. For example, 'CAMEO_INTL_2015' can be split into 'WEALTH' and 'LIFE_CYCLE'. Similarly, we can split 'PRAEGENDE_JUGENDJAHRE' into 'MOVEMENT' AND 'GENERATION'<jupyter_code>def mixed_categories(df):
    '''
    Spliting mixed attributes into individual features;
    Input:
    df: DataFrame to be processed
    
    Output:
    df
    '''
    #spliting 'CAMEO_INTL_2015' can be split into 'WEALTH' and 'LIFE_CYCLE'
    df['WEALTH']=df['CAMEO_INTL_2015'].apply(lambda x:x/10)
    df['LIFE_CYCLE']=df['CAMEO_INTL_2015'].apply(lambda x:x%10)
    
    #split 'PRAEGENDE_JUGENDJAHRE' into 'MOVEMENT' 
    mainstream=[1.0, 3.0, 5.0, 8.0, 10.0, 12.0, 14.0]
    avantgarde=[2.0, 4.0, 6.0, 7.0, 9.0, 11.0, 13.0, 15.0]
    
    main=df['PRAEGENDE_JUGENDJAHRE'].isin([1.0, 3.0, 5.0, 8.0, 10.0, 12.0, 14.0])
    avar=df['PRAEGENDE_JUGENDJAHRE'].isin([2.0, 4.0, 6.0, 7.0, 9.0, 11.0, 13.0, 15.0])
    
    df.loc[main,'MOVEMENT']=1.0
    df.loc[avar,'MOVEMENT']=2.0
    
    #Dropping the original features and categorical features which are too detailed
    df=df.drop(['CAMEO_INTL_2015','PRAEGENDE_JUGENDJAHRE', 'EINGEFUEGT_AM','D19_LETZTER_KAUF_BRANCHE'],axis=1,inplace=False)
    
    #Converting 'OST_WEST_KZ' into numerical values
    df['OST_WEST_KZ'] = df['OST_WEST_KZ'].replace({'O':1.0, 'W':2.0})
    
    #Getting dummies values for 'CAMEO_DEU_2015'
    new_df=pd.get_dummies(df,columns=['CAMEO_DEU_2015'])
    
    new_df=new_df.drop(['CAMEO_DEU_2015_XX'],axis=1,inplace=False)
    
    return new_df
    
    
azdias=mixed_categories(azdias)<jupyter_output><empty_output><jupyter_text>Imputing and Standardization of each column<jupyter_code>azdias_col=azdias.columns
imputer=SimpleImputer()
scaler=StandardScaler()
azdias=imputer.fit_transform(azdias)
azdias_std=scaler.fit_transform(azdias)
azdias_transformed = pd.DataFrame(azdias_std, columns = azdias_col)
azdias_transformed.head()
#to free up the space
del azdias
del azdias_transformed<jupyter_output><empty_output><jupyter_text>PCA<jupyter_code>n_components=int((azdias_std.shape[1])/2)

pca = PCA(n_components)
azdias_pca = pca.fit_transform(azdias_std)
index=np.arange(n_components)
value=pca.explained_variance_ratio_

plt.figure(figsize=(10, 8))
ax = plt.subplot()
cum_value = np.cumsum(value)
ax.bar(index, value)
ax.plot(index, cum_value)
for i in range(n_components):
    ax.xaxis.set_tick_params(width=0)
    ax.yaxis.set_tick_params(width=3, length=12)

    ax.set_xlabel("No of Principal Components")
    ax.set_ylabel("Variance (%)")
    plt.title('Variance Per Principal Component')
sum(pca.explained_variance_ratio_)
n_components
#Converting the PCA components to the features of the clean dataset. 
#Mapping each feature to the principal components

dim =['Dimension' +str(i) for i in range(1,len(pca.components_)+1)]
comp_dist=pd.DataFrame(np.round(pca.components_,5),columns=azdias_col)
comp_dist.index=dim
comp_dist.head(3)
print('Principal Component 1: ')
print('Top 5 features: ')
print(comp_dist.iloc[0].sort_values(ascending=False)[:5])
print('Bottom 5 features: ')
print(comp_dist.iloc[0].sort_values(ascending=False)[-5:])
print('Variance Explained: ',pca.explained_variance_ratio_[0]*100)
print('Principal Component 2: ')
print('Top 5 features: ')
print(comp_dist.iloc[1].sort_values(ascending=False)[:5])
print('Bottom 5 features: ')
print(comp_dist.iloc[1].sort_values(ascending=False)[-5:])
print('Variance Explained: ',pca.explained_variance_ratio_[1]*100)
print('Principal Component 3: ')
print('Top 5 features: ')
print(comp_dist.iloc[2].sort_values(ascending=False)[:5])
print('Bottom 5 features: ')
print(comp_dist.iloc[2].sort_values(ascending=False)[-5:])
print('Variance Explained: ',pca.explained_variance_ratio_[2]*100)
azdias_pca.shape<jupyter_output><empty_output><jupyter_text>After applying PCA to the azdias dataFrame, we can say that around 181 features combined give us 90% variance of the entire dataset. The first 3 components of PCA are as follows:1) 1st Principal Component: The following are the top five features for this dimension:**Feature_Name    Variance_Explained****MOBI_REGIO       0.14437PLZ8_ANTG1     0.14269KBA13_ANTG1    0.14213KBA05_ANTG1    0.13422MOBI_RASTER    0.13274**The following are the bottom five features for this dimension:**Feature_Name   Variance_Explained****PLZ8_ANTG4     -0.13624KBA13_BAUMAX   -0.13700KBA13_ANTG4    -0.13775KBA13_ANTG3    -0.14113PLZ8_ANTG3     -0.14163**Total Variance explained by 1st component: **7.777**2) 2nd Principal Component: The following are the top five features for this dimension:**Feature_Name   Variance_Explained****KBA13_HERST_BMW_BENZ           0.18315KBA13_MERCEDES                 0.15690KBA13_SEG_OBEREMITTELKLASSE    0.15629KBA13_BMW                      0.15229KBA13_SITZE_4                  0.14574**The following are the bottom five features for this dimension:**Feature_Name   Variance_Explained****KBA13_KMH_180          -0.12094KBA13_HALTER_25        -0.12098KBA13_SEG_KLEINWAGEN   -0.13142KBA13_KMH_140_210      -0.13198KBA13_SITZE_5          -0.15013**Total Variance explained by 2nd component: **4.88**3) 3rd Principal Component: The following are the top five features for this dimension:**Feature_Name   Variance_Explained****ONLINE_AFFINITAET     0.18171D19_GESAMT_ANZ_24     0.16518D19_GESAMT_ANZ_12     0.15901D19_VERSAND_ANZ_24    0.15094ALTER_HH              0.15035**The following are the bottom five features for this dimension:**Feature_Name   Variance_Explained****CJT_TYP_5                  -0.15228CJT_TYP_6                  -0.15275D19_VERSAND_ONLINE_DATUM   -0.15482CJT_TYP_4                  -0.15493D19_GESAMT_ONLINE_DATUM    -0.16054**Total Variance explained by 3rd component: **4.32**<jupyter_code>comp_dist.iloc[0].sort_values(ascending=False)[:10].plot(kind='barh')
plt.title('1st Component Important Features')
comp_dist.iloc[1].sort_values(ascending=False)[:10].plot(kind='barh')
plt.title('2nd Component Important Features')
comp_dist.iloc[2].sort_values(ascending=False)[:10].plot(kind='barh')
plt.title('3rd Component Important Features')<jupyter_output><empty_output><jupyter_text>Clustering Azdias_df<jupyter_code>score=[]
centers=np.arange(10,31)

for i in centers:
    
    k_mean=MiniBatchKMeans(n_clusters=i).fit(azdias_pca)
    #Append the score with different centers
    score.append(np.abs(k_mean.score(azdias_pca)))
plt.plot(centers,score,marker='*')
plt.xlabel('No of centers')
plt.ylabel('SSE')
plt.title('Sum of Squared distance for different no of centers')
kmeans=KMeans(n_clusters=30)
azdias_kmeans=kmeans.fit_predict(azdias_pca)
azdias_cluster=pd.DataFrame(np.round(azdias_pca,5),columns=dim)
azdias_cluster.insert(loc=0,column='Cluster',value=azdias_kmeans)
azdias_cluster.head()
del azdias_pca
del azdias_kmeans<jupyter_output><empty_output><jupyter_text>Applying these changes to the customer_dataset<jupyter_code>#Data loading
customers = pd.read_csv('../capstone_data/Udacity_CUSTOMERS_052018.csv',low_memory=False)
customers=customers.drop(['CUSTOMER_GROUP', 'ONLINE_PURCHASE', 'PRODUCT_GROUP'], axis=1)
#CLeaning Customers
customers=Map_unknown_to_NAN(customers,attribute_dict)
customers.drop(drop_col_list,axis=1,inplace=True)
customers=cat_to_num(customers)
customers=mixed_categories(customers)
#customers col
customers_col=customers.columns
#Imputing and standardization
customers_imp=imputer.transform(customers)
customers_std=scaler.transform(customers_imp)
#Checking that both have same columns: 
sum(customers_col==azdias_col)#should be equal to 363 (features after cleaning)
#Applying PCA and Kmeans to customers
customers_pca=pca.transform(customers_std)
customers_kmeans=kmeans.predict(customers_pca)
customers_cluster=pd.DataFrame(np.round(customers_pca,5),columns=dim)
customers_cluster.insert(loc=0,column='Cluster',value=customers_kmeans)
customers_cluster.head()
del customers_pca
del customers_kmeans<jupyter_output><empty_output><jupyter_text>Customer Segmentation<jupyter_code>#Creating a df from the difference of both the clusters
temp=(azdias_cluster['Cluster'].value_counts()/azdias_cluster.shape[0]).reset_index().sort_values(by=['index'])
azdias_count=np.array(temp['Cluster'])
cust_temp=(customers_cluster['Cluster'].value_counts()/customers_cluster.shape[0]).reset_index().sort_values(by=['index'])
customers_count=np.array(cust_temp['Cluster'])
#PLotting graph
x_range=temp['index']
bar_width=0.5

fig,ax =plt.subplots()

ax.bar(x_range,azdias_count,bar_width,label='Azdias')
ax.bar(x_range+bar_width,customers_count,bar_width,label='Customer')
plt.legend()
plt.title('Proportion of general and customers in each clustor')
plt.xlabel('Number of cluster')
plt.ylabel('Proportion')
diff_count=customers_count-azdias_count
#Creating df from difference of both clusters
diff_count=pd.DataFrame(diff_count,columns=['Count'])

sns.barplot(diff_count.index,diff_count['Count'])
plt.title('Difference in proportion between customers and azdias')
plt.xlabel('Number of cluster')
plt.ylabel('Diff in Proportion')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11377283357562733, 'fraction_numerical': 0.03893559193477024, 'mean_word_length': 5.261179263949347, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '276137', 'license': 'CNRI-Python', 'n_tokens_mistral': 6074, 'n_tokens_neox': 5612, 'n_words': 1141, 'path': 'Customer_Segmentation.ipynb', 'repo_name': 'RuchitDoshi/Avarto_Customer_Segmentation', 'size': '15821'}