<jupyter_start><jupyter_text>What is Customer segmentation?  Customer segmentation is the process of dividing customers into groups based on common characteristics so companies can market to each group effectively and appropriately.  In business to customer marketing, companies often segment customers according to demographics that include:  * Age* Gender  * Marital status  * Location (urban, suburban, rural)  Set-up<jupyter_code># Connect to google drive
from google.colab import drive
drive.mount('/content/gdrive')
# install pywaffle
!pip install pywaffle
# import libraries
import pandas as pd
import os, re
import datetime as dt
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from pywaffle import Waffle
from matplotlib import gridspec
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
# define a func to plit waffle chart
def plot_waffle_chart_(df, metric, agg, title_, group='sub_segment'):
  dict_revenue = dict(round(df.groupby(group).agg({metric:agg}),0)[metric])
  plt.figure( FigureClass=Waffle, rows = 5, columns =10, values = dict_revenue,
             labels =[(x,y) for x,y in dict_revenue.items()], 
             legend ={'loc':'lower left', 'bbox_to_anchor':(1,0)},
             figsize= (15,5)
             )
  plt.title(title_)
# define a funtion to convert datatype
def convert_to_dt(df, cols):
  for col in cols:
    df[col] = pd.to_datetime(df[col], format= '%Y-%m-%d').dt.date
# path to data
path = '/content/gdrive/MyDrive/brazilian_ecommerce'<jupyter_output><empty_output><jupyter_text>Data<jupyter_code># read data
df_orders = pd.read_csv(os.path.join(path,'df_order_items_products_customers.csv'))
df_orders.head()
# overall info
df_orders.info()
# converting date
cols = ['order_purchase_timestamp','order_approved_at','order_delivered_carrier_date',
                       'order_delivered_customer_date','order_estimated_delivery_date','shipping_limit_date' ]
convert_to_dt(df_orders, cols)
df_orders.head()
# define a sub-dataframe
df_segment = df_orders[['customer_unique_id','order_id','order_status','order_purchase_timestamp',
    'product_id','quantity','price','product_category_name']].copy()
df_segment.head()
# check how many status
df_segment['order_status'].value_counts()
# using only delivered status
df_segment = df_segment[df_segment['order_status']=='delivered']
# calculate the total price for an order line
df_segment['total_payment'] = df_segment['quantity'] * df_segment['price']<jupyter_output><empty_output><jupyter_text>Customer segmentation  segmentation based on recency and payment<jupyter_code># last day of dataset
df_segment['order_purchase_timestamp'].max()
# assume now is 30/08
now = dt.date(2018, 8, 30)
# calculate recency, frequency, monetary
df_segmented_customers = df_segment.groupby('customer_unique_id').agg({'order_purchase_timestamp': lambda date:(now-date.max()).days,
                                                                       'order_id':pd.Series.nunique,
                                                                       'total_payment': ['sum','mean'],
                                                                       'quantity':'sum'}).reset_index()  
# renaming columns
df_segmented_customers.columns = ['customer_unique_id','recency','frequency','total_payment','avg_payment','quantity']
df_segmented_customers.head()
# Defining time range conditions for classifying customers
total_time_range = str(df_segment["order_purchase_timestamp"].max() - df_segment["order_purchase_timestamp"].min())
total_time_range
# Splitting total time range in 4 time periods
period = int(re.sub(r'\s+days.*', '', total_time_range))/4
period
# activity classes
class_inactive = df_segmented_customers['recency'] > period*3
class_cold = (df_segmented_customers['recency'] > period*2)&(df_segmented_customers['recency']<=period*3)
class_hot = (df_segmented_customers['recency'] > period)&(df_segmented_customers['recency']<=period*2)
class_active = df_segmented_customers['recency'] <= period
# add new columns
# customers who did not make any purchases or their last purchase was more than three times of the period defined as a parameter
df_segmented_customers.loc[class_inactive, "segment"]= "inactive" 
# customers that their last purchase was greater than twice period and less than or equal to the three times the period
df_segmented_customers.loc[class_cold, 'segment'] = 'cold'
# customers that their last purchase was greater than the period and less than or equal to the twice period
df_segmented_customers.loc[class_hot,'segment'] = 'hot'
# customers that their last purchase was less than or equal to the period
df_segmented_customers.loc[class_active, 'segment'] = 'active'

df_segmented_customers.head()

# define payment range conditions for classifying customers
median_payment = df_segmented_customers['avg_payment'].median()
median_payment
# sub classes
class_inactive_low = df_segmented_customers['segment'] =='inactive'
class_cold_low = (df_segmented_customers['segment']=='cold') &(df_segmented_customers['avg_payment']<median_payment)
class_cold_high = (df_segmented_customers['segment']=='cold') &(df_segmented_customers['avg_payment']>=median_payment)

class_hot_low = (df_segmented_customers['segment'] == 'hot') & (df_segmented_customers['avg_payment'] < median_payment)
class_hot_high = (df_segmented_customers['segment'] == 'hot') & (df_segmented_customers['avg_payment'] >= median_payment)

class_active_low = (df_segmented_customers['segment'] == 'active')& (df_segmented_customers['avg_payment'] < median_payment)
class_active_high = (df_segmented_customers['segment'] == 'active')& (df_segmented_customers['avg_payment'] >= median_payment)

# add sub classes to dataframe
# customers who did not make any purchases or their last purchase was more than three times of the period defined as a parameter
df_segmented_customers.loc[class_inactive_low, "sub_segment"] = "inactive"
# this segment is equal to cold and have purchases less than the median defined as a parameter 
df_segmented_customers.loc[class_cold_low, "sub_segment"] = "cold_low"
# this segment is equal to cold and have purchases greater than or equal to the median defined as a parameter
df_segmented_customers.loc[class_cold_high, "sub_segment"] = "cold_high"
# this segment is equal to hot and have purchases less than the median defined as a parameter
df_segmented_customers.loc[class_hot_low, "sub_segment"] = "hot_low"
# this segment is equal to hot and have purchases greater than or equal to the median defined as a parameter
df_segmented_customers.loc[class_hot_high, "sub_segment"] = "hot_high"
# this segment is equal to active and have purchases less than the median defined as a parameter
df_segmented_customers.loc[class_active_low, "sub_segment"] = "active_low"
# this segment is equal to active and have purchases greater than or equal to the median defined as a parameter
df_segmented_customers.loc[class_active_high, "sub_segment"] = "active_high"

df_segmented_customers.head()
# number of customers per sub-segment
plot_waffle_chart_(df_segmented_customers,'customer_unique_id', 'count', 'Number of customers by Sub-Segment')
# total orders placed by sub-segment
plot_waffle_chart_(df_segmented_customers, 'frequency', 'sum', 'Orders by Sub-Segment')
# total revenue by sub-segment
plot_waffle_chart_(df_segmented_customers, 'total_payment', 'sum', 'Revenue by Sub-Segment')<jupyter_output><empty_output><jupyter_text>From the graph we can see that:* The strongest class of the 7 generated sub-segments is active customers with high spending, followed by hot customers which high spending, which is a good sign for the business.  * Active customers are the biggest groups* Active customers with high and low spending placed the most orders, followed by hot customers* Inactive customers accounted for only a small proportion.<jupyter_code># save file
df_segmented_customers.to_csv(os.path.join(path,'df_segmented_customers.csv'), encoding='utf-8-sig',index=False)<jupyter_output><empty_output><jupyter_text>RFM RFM analysis is a customer segmentation technique that use past purchase behaviour to segment customers.  The name RFM stands for Recency, Frequency, and Monetary value. * Recency: the number of days since the customer's last purchase  * Frequency: the number of times the customer has purchased in the timeframe being analyzed  * Monetary: the total amount spent by the customer in the same period<jupyter_code># read data
df_rfm = pd.read_csv(os.path.join(path,'df_segmented_customers.csv'))
df_rfm.head()
# plot chart
# define figure
fig = plt.figure(constrained_layout=True, figsize=(12,10))
# define axis
gs = gridspec.GridSpec(3, 1, figure=fig)
ax1 = fig.add_subplot(gs[0, 0])
ax2 = fig.add_subplot(gs[1, 0])
ax3 = fig.add_subplot(gs[2, 0])
# plot recency
sns.histplot(df_rfm['recency'], kde=True, ax = ax1)
ax1.set_title('Recency', fontsize = 12)
ax1.set_xlabel('Days since last order',fontsize= 12)
ax1.spines['top'].set_visible(False)
ax1.spines['right'].set_visible(False)
# plot frequency
sns.histplot(df_rfm['frequency'], kde= False, ax = ax2, bins=80)
ax2.set_xlim([0,5])
ax2.set_title('Frequency',fontsize= 12)
ax2.set_xlabel('Number of Orders', fontsize =12)
ax2.spines['top'].set_visible(False)
ax2.spines['right'].set_visible(False)
# plot monetary
sns.histplot(df_rfm['total_payment'], ax = ax3, bins=1000)
ax3.set_title('Monetary',fontsize = 12)
ax3.set_xlabel('Total purchase value', fontsize =12)
ax3.set_xlim([0,500])
ax3.spines['top'].set_visible(False)
ax3.spines['right'].set_visible(False)

plt.show()<jupyter_output><empty_output><jupyter_text>From the charts above, we can see that  * There is a peak at round about 270 days due to black friday* There are many customers which only placed one order  * The majority of the prices is in the lower range of up to 100  Calculation<jupyter_code># frequencies:
df_rfm['frequency'].unique()
# define a funct
def assign_frequency(x):
  if x>=7:
    return 4
  elif x>=4:
    return 3
  elif x>=2:
    return 2
  else:
    return 1

# add new column
df_rfm['R'] = pd.qcut(x=df_rfm['recency'], q=4, labels=range(4,0,-1))
df_rfm['F'] = df_rfm['frequency'].apply(assign_frequency)
df_rfm['M'] = pd.qcut(x=df_rfm['total_payment'], q =4, labels = range(1,5))
# define- and assign label to dataframe
df_rfm['segment_RFM'] = df_rfm['R'].astype(str)+ df_rfm['F'].astype(str) + df_rfm['M'].astype(str)
df_rfm['score_rfm'] = df_rfm[['R','F','M']].sum(axis=1)

df_rfm
# calculate mean by score_rfm
df_rfm.groupby('score_rfm').mean()[['recency','frequency','total_payment']]
# define a func
def rfm_assiner(df):
  if (int(df['segment_RFM'])>=434) or (df['score_rfm']>=9):
    return 'Best customer'
  elif (df['score_rfm'] >=8) and (df['M']==4):
    return 'Big Spender'
  elif (df['score_rfm'] >= 6) and (df['F'] >= 2):
    return 'Loyalist'
  elif (int(df['segment_RFM']) >= 231) or (df['score_rfm'] >= 6):
    return 'Potential Loyalists'
  elif ((int(df['segment_RFM']) >= 121) and (df['R'] == 1)) or df['score_rfm'] == 5:
    return 'Almost Lost'
  elif (df['score_rfm'] >= 4) and (df['R'] == 1):
    return 'Hibernating'
  else:
    return 'Lost Customer'
# add new column
df_rfm['customer_rfm_segment'] = df_rfm.apply(rfm_assiner, axis=1)
df_rfm.head()

# approach suggestion
dict_rfm_tiers = {'Best customer': 'Personalized communication, offer loyalty program, no promotional offers needed',
                  'Big Spender': 'Make them feel valued and offer quality products, encourage to stick with brands', 
                  'Loyalist': 'Offer loyalty program', 
                  'Potential Loyalists': 'Recommend products and offer discounts',
                  'Almost Lost': 'Try to win them with limited sales promotions', 
                  'Hibernating': 'Make great offers with big discounts', 
                  'Lost Customer': 'Do not spent much effort and money to win them'}
df_rfm['marketing_approach'] = df_rfm['customer_rfm_segment'].apply(lambda x: dict_rfm_tiers[x])
df_rfm
# Vizualize segment sizes with the waffe chart
plot_waffle_chart_(df_rfm, 'customer_unique_id', 'count', 'RFM segments', 'customer_rfm_segment')<jupyter_output><empty_output><jupyter_text>* The biggest RFM segment is the group of Potential loyalists. These are customers that need to targeted with more offers and discounts in order to make them buy more.  * Some effort needs to be spent to target hibernating customers and customers which are almost lost<jupyter_code># table
df_tbl_rfm = df_rfm.groupby('customer_rfm_segment').agg(
    Count = ('customer_unique_id', 'count'),
    Recency = ('recency', 'mean'),
    Frequency = ('frequency', 'mean'),
    Monetary = ('total_payment', 'mean'),
    Strategy = ('marketing_approach', 'unique'),
).round(1)
df_tbl_rfm<jupyter_output><empty_output><jupyter_text>K-Means clustering<jupyter_code># create a dataframe using for clustering
df_rfm_kmean = df_rfm[['customer_unique_id','recency', 'frequency', 'total_payment']]
df_rfm_kmean = df_rfm_kmean.set_index('customer_unique_id')
scaler = StandardScaler() 
scaler.fit(df_rfm_kmean) 
df_rfm_norm = scaler.transform(df_rfm_kmean)
df_rfm_norm = pd.DataFrame(data=df_rfm_norm, index=df_rfm_kmean.index, columns=df_rfm_kmean.columns)
df_rfm_norm.head()
# find best number of clusters
sse={}
for k in range(1, 10):
  kmeans = KMeans(n_clusters=k, max_iter=1000).fit(df_rfm_norm)
  sse[k] = kmeans.inertia_
plt.figure(figsize=(15,5))
plt.plot(list(sse.keys()), list(sse.values()), marker= 'o')
plt.xlabel('Number of clusters "K"')
plt.ylabel('Sum of Squared Errors')
plt.title('Elbow Method - Clusters vs. Sum of Squared Errors');
plt.show()<jupyter_output><empty_output><jupyter_text>The Ellbow Method indicates that a cluster number of 4 is a suitable value.<jupyter_code># how this clustering looks like
k_means = KMeans(n_clusters=4, random_state=42)
k_means.fit(df_rfm_norm)
cluster = k_means.labels_
df_rfm_4 = df_rfm.assign(Cluster=cluster)
df_group_4 = df_rfm_4.groupby('Cluster').agg({
    'recency': 'mean',
    'frequency': 'mean',
    'total_payment': ['mean', 'count']
  }).round(1)
df_group_4
df_rfm_4.head()
# save file
df_rfm_4.to_csv(os.path.join(path,'df_rfm_clustering.csv'), encoding='utf-8-sig',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11774576982240247, 'fraction_numerical': 0.010348203048524681, 'mean_word_length': 5.718177548144669, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '452756', 'license': 'MIT', 'n_tokens_mistral': 4764, 'n_tokens_neox': 4575, 'n_words': 1238, 'path': '4_Customer_Segmentation_RFM.ipynb', 'repo_name': 'thaitruong018/e_commerce_analysis', 'size': '14302'}