<jupyter_start><jupyter_text>0.0 Imports<jupyter_code>import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from IPython.display     import HTML
from sklearn.preprocessing import MinMaxScaler
from sklearn.cluster import KMeans
from yellowbrick.cluster import KElbowVisualizer
from yellowbrick.cluster import SilhouetteVisualizer
from sklearn.metrics import silhouette_score, calinski_harabasz_score, davies_bouldin_score<jupyter_output><empty_output><jupyter_text>0.1 Helper Functions<jupyter_code>def jupyter_settings():
    %matplotlib inline
    %pylab inline
    
    plt.style.use( 'bmh' )
    plt.rcParams['figure.figsize'] = [18, 9]
    plt.rcParams['font.size'] = 24

    display( HTML( '<style>.container { width:100% !important; }</style>') )
    pd.options.display.max_columns = None
    pd.options.display.max_rows = None
    pd.set_option( 'display.expand_frame_repr', False )
    pd.set_option('display.float_format', lambda x: '%.2f' % x)
    sns.set()
    
jupyter_settings()<jupyter_output>Populating the interactive namespace from numpy and matplotlib<jupyter_text>0.2 Load Data<jupyter_code>df_raw = pd.read_csv('../data/CC GENERAL.csv')<jupyter_output><empty_output><jupyter_text>0.3 Data Dictionary - CUSTID : Identification of Credit Card holder (Categorical)- BALANCE : Balance amount left in their account to make purchases - BALANCEFREQUENCY : How frequently the Balance is updated, score between 0 and 1 (1 = frequently updated, 0 = not frequently updated)- PURCHASES : Amount of purchases made from account- ONEOFFPURCHASES : Maximum purchase amount done in one-go- INSTALLMENTSPURCHASES : Amount of purchase done in installment- CASHADVANCE : Cash in advance given by the user- PURCHASESFREQUENCY : How frequently the Purchases are being made, score between 0 and 1 (1 = frequently purchased, 0 = not frequently purchased)- ONEOFFPURCHASESFREQUENCY : How frequently Purchases are happening in one-go (1 = frequently purchased, 0 = not frequently purchased)- PURCHASESINSTALLMENTSFREQUENCY : How frequently purchases in installments are being done (1 = frequently done, 0 = not frequently done)- CASHADVANCEFREQUENCY : How frequently the cash in advance being paid- CASHADVANCETRX : Number of Transactions made with "Cash in Advanced"- PURCHASESTRX : Numbe of purchase transactions made- CREDITLIMIT : Limit of Credit Card for user- PAYMENTS : Amount of Payment done by user- MINIMUM_PAYMENTS : Minimum amount of payments made by user- PRCFULLPAYMENT : Percent of full payment paid by user- TENURE : Tenure of credit card service for user  1.0 Data Description<jupyter_code>df1 = df_raw.copy()
df1.head()<jupyter_output><empty_output><jupyter_text>1.1 Rename Columns<jupyter_code>df1.columns = df1.columns.str.lower()<jupyter_output><empty_output><jupyter_text>1.2 Data Dimensions<jupyter_code>df1.shape<jupyter_output><empty_output><jupyter_text>O Dataset possui 8950 linhas e 18 colunas  1.3 Data Types<jupyter_code>df1.dtypes<jupyter_output><empty_output><jupyter_text>1.4 Check Na<jupyter_code>df1.isna().sum()*100/len(df1)<jupyter_output><empty_output><jupyter_text>1.5 Replace Na<jupyter_code>df1 = df1.dropna()<jupyter_output><empty_output><jupyter_text>1.6 Change Data Types No momento nenhuma mudança será feita  1.7 Descriptive Statistics  2.0 Feature Engineering<jupyter_code>df2 = df1.copy()<jupyter_output><empty_output><jupyter_text>2.1 Feature Creation<jupyter_code># Gross Revenue (Faturamento)
df2['gross_revenue'] = df2['purchases'] * df2['purchases_trx']<jupyter_output><empty_output><jupyter_text>3.0 Variable Filtering<jupyter_code>df3 = df2.copy()
df3 = df3.drop_duplicates(subset=['cust_id'])<jupyter_output><empty_output><jupyter_text>4.0 EDA<jupyter_code>df4 = df3.copy()<jupyter_output><empty_output><jupyter_text>5.0 Data Preparation<jupyter_code>df5 = df4.copy()<jupyter_output><empty_output><jupyter_text>5.1 Rescaling<jupyter_code>mms = MinMaxScaler()
df5 = df5.drop('cust_id', axis=1)
cols = ['balance', 'balance_frequency', 'purchases', 'oneoff_purchases',
       'installments_purchases', 'cash_advance', 'purchases_frequency',
       'oneoff_purchases_frequency', 'purchases_installments_frequency',
       'cash_advance_frequency', 'cash_advance_trx', 'purchases_trx',
       'credit_limit', 'payments', 'minimum_payments', 'prc_full_payment',
       'tenure', 'gross_revenue']
df5[cols] = mms.fit_transform(df5[cols].values)<jupyter_output><empty_output><jupyter_text>6.0 Feature Selection<jupyter_code>df6 = df5.copy()<jupyter_output><empty_output><jupyter_text>7.0 Hyperparameter Fine-Tunning<jupyter_code>X = df6.copy()
clusters = [2,3,4,5,6,7,8,9,10]<jupyter_output><empty_output><jupyter_text>7.1 Calinski Harabasz Index<jupyter_code>kmeans = KElbowVisualizer(KMeans(), k=clusters, metric='calinski_harabasz', timing=False)
kmeans.fit(X)
kmeans.show();<jupyter_output><empty_output><jupyter_text>7.2 Silhouette Score<jupyter_code>kmeans = KElbowVisualizer(KMeans(), k=clusters, metric='silhouette', timing=False)
kmeans.fit(X)
kmeans.show();<jupyter_output><empty_output><jupyter_text>7.2.1 Silhouette Analysis<jupyter_code>fig,ax = plt.subplots(5,2, figsize=(25,18))

for k in clusters:
    km = KMeans(n_clusters=k, init='random', max_iter=100, random_state=42)
    q, mod = divmod(k,2)
    visualizer = SilhouetteVisualizer(km, colors='yellowbrick', ax=ax[q-1][mod])
    visualizer.fit(X)
    visualizer.finalize()
    
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>7.3 Davies-Bouldin Index<jupyter_code>score = []

for k in clusters:
    # model definiton
    kmeans = KMeans(init='random', n_clusters=k, n_init=10, max_iter=300, random_state=42 )
    
    # model training
    kmeans.fit(X)
    
    # clustering
    labels =kmeans.labels_
    
    # validation
    score.append(davies_bouldin_score(X, labels))
    
# plot wss
plt.plot(clusters, score, linestyle='--', marker='o', color='k')
plt.xlabel('Number of cluster');
plt.ylabel('Davies-Bouldin Score');
plt.title('Davies-Bouldin Score vs Number of clusters');<jupyter_output><empty_output><jupyter_text>8.0 Model Training  8.1 KMeans<jupyter_code># model definition
k = 10
kmeans = KMeans(init='random', n_clusters=k, n_init=10, max_iter=300)

# model traiing
kmeans.fit(X)

# clustering
labels =kmeans.labels_<jupyter_output><empty_output><jupyter_text>8.2 Cluster Validation<jupyter_code>## Calinski Harabasz Score
print('Calinski Harabasz Score Value: {}'.format(calinski_harabasz_score(X, labels)))

## SS (Silhouette Score)
print('SS value: {}'.format(silhouette_score(X, labels, metric='euclidean')))

## Davies-Bouldin Score
print('Davies Bouldin Score Value: {}'.format(davies_bouldin_score(X, labels)))<jupyter_output>Calinski Harabasz Score Value: 3011.429888816202
SS value: 0.33666641100114914
Davies Bouldin Score Value: 1.0450748604766864<jupyter_text>9.0 Cluster Analysis<jupyter_code>df9 = df4.copy()

df9['cluster'] = labels
df9.sample(5)<jupyter_output><empty_output><jupyter_text>9.1 Cluster Profile<jupyter_code># Number of customers
df_cluster = df9[['cust_id','cluster']].groupby('cluster').count().reset_index()
df_cluster['perc_customer'] = 100*(df_cluster['cust_id'] / df_cluster['cust_id'].sum())

# Avg gross revenue
df_avg_gross_revenue = df9[['gross_revenue','cluster']].groupby('cluster').mean().reset_index()
df_cluster = pd.merge(df_cluster, df_avg_gross_revenue, how='inner', on='cluster')
df_cluster
df9.sample(4)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11990889603429797, 'fraction_numerical': 0.02947481243301179, 'mean_word_length': 6.907838983050848, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '368715', 'license': 'MIT', 'n_tokens_mistral': 2852, 'n_tokens_neox': 2717, 'n_words': 528, 'path': 'notebooks/c01-customer-segmentation-end_to_end.ipynb', 'repo_name': 'Leonardodsch/customer-segmentation', 'size': '7464'}