<jupyter_start><jupyter_text>Clustering on the Cineplex Concessions- <NAME>- October 2020- For: MMA/GMMA/MMAI 869This Notebook will do some basic clustering on the Cineplex members.<jupyter_code>import pandas as pd
import numpy as np

from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"<jupyter_output><empty_output><jupyter_text>Load in the data files<jupyter_code>version_str = "_v3"
input_dir = "clean_v3/"
visits = pd.read_csv(input_dir+'visits'+version_str+'.csv')
# Extract some info about the date
visits['Visit_Date'] = pd.to_datetime(visits['Visit_Date'])
visits['Visit_Date_DOW'] = visits['Visit_Date'].dt.day_name()
visits['Visit_Date_Week'] = visits['Visit_Date'].dt.strftime('%U')
visits['Visit_Date_Month'] = visits['Visit_Date'].dt.month_name()
visits['Visit_Date_IsWeekend'] = np.where(visits['Visit_Date'].dt.weekday < 5, 0, 1)
visits['Visit_Date_IsTuesday'] = np.where(visits['Visit_Date'].dt.weekday == 1, 1, 0)
visits.info()
items = pd.read_csv(input_dir+'items'+version_str+'.csv', encoding = "ISO-8859-1")
items.info()
visits_items = pd.read_csv(input_dir+'visit_items'+version_str+'.csv', encoding = "ISO-8859-1")
visits_items.info()
films = pd.read_csv(input_dir+'films'+version_str+'.csv', encoding = "ISO-8859-1")
films.info()
experience_type = pd.read_csv(input_dir+'experience_type'+version_str+'.csv', encoding = "ISO-8859-1")
experience_type.info()
sales_channels = pd.read_csv(input_dir+'sales_channels'+version_str+'.csv', encoding = "ISO-8859-1")
sales_channels.info()
timeslice = pd.read_csv(input_dir+'timeslice'+version_str+'.csv', encoding = "ISO-8859-1")
timeslice.info()
locations = pd.read_csv(input_dir+'locations'+version_str+'.csv', encoding = "ISO-8859-1")
locations.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 496 entries, 0 to 495
Data columns (total 13 columns):
 #   Column                     Non-Null Count  Dtype 
---  ------                     --------------  ----- 
 0   Location_ID                496 non-null    int64 
 1   Location_Name              495 non-null    object
 2   Location_Short_Name        492 non-null    object
 3   Location_Interactive_Name  494 non-null    object
 4   Address1                   483 non-null    object
 5   City                       489 non-null    object
 6   Province_CD                496 non-null    object
 7   Postal_Code                438 non-null    object
 8   Country_CD                 496 non-null    object
 9   Is_VIP                     496 non-null    int64 
 10  Is_Outtakes                496 non-null    int64 
 11  Is_Licensed_Bar            496 non-null    int64 
 12  Is_Licensed_Lounge         496 non-null    int64 
dtypes: int64(5), object(8)
memory usage: 50.5+ KB<jupyter_text>VisitsVisits is the main "fact" table in the dataset. Let's first merge (join) it with all the dimension tables, and then we can summarize how often everything occurs.<jupyter_code>visits = visits.merge(films, how="left")
visits = visits.merge(experience_type, how="left")
visits = visits.merge(sales_channels, how="left")
visits = visits.merge(timeslice, left_on="Visit_TimeSlice_ID", right_on="TimeSlice_ID", how="left")
visits = visits.merge(locations, how="left")
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.float_format', lambda x: '%.1f' % x):
    display(visits.head())
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.float_format', lambda x: '%.1f' % x):
    visits.describe(include=['O']).T
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.float_format', lambda x: '%.1f' % x):
    visits.describe().T
visits['CINEPLEX_D_Membership_ID'].nunique()<jupyter_output><empty_output><jupyter_text>Filter for Kiddie Tray Customers<jupyter_code>visits.shape
visits_kt=visits[visits["Has_US_Is_Kiddietray"]==1].drop("Has_US_Is_Kiddietray",axis=1)
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.float_format', lambda x: '%.1f' % x):
    visits_kt.head()
visits_kt.info()<jupyter_output><empty_output><jupyter_text>Per-Member Visit StatsFor each member, let's summarize their visit behavior.<jupyter_code># Define my own version of the 'mode' function, which will just return the first mode (in case of tie) and thus avoid an annoying error from groupby (whcih always wants just one value)
my_mode = lambda x: pd.Series.mode(x)[0]

members_visits = visits_kt.groupby('CINEPLEX_D_Membership_ID').agg(
    {
        'Visit_Date': ['count'],
#         'Visit_Date_DOW': [my_mode],
        'Visit_Date_IsTuesday': ['mean'],
        'Visit_Date_IsWeekend': ['mean'],
#         'TimeSlice_Hour_24': ['mean', 'max'],
        'TimeSlice_Description': my_mode,
        
        #'Location_ID': [pd.Series.nunique, my_mode],
#         'City': [my_mode],
        'Province_CD': [ my_mode],
        
        'Auditorium_Experience_ID': [my_mode],
#         'Seating_Experience_ID': [pd.Series.nunique, my_mode],
#         'Sales_Channel_ID': [my_mode],
        'Media_Class_Description': [my_mode],
#         'Performance_Experience_ID': [pd.Series.nunique, my_mode],
#         'Performance_Type_ID': [my_mode],

#         'Film_ID': [pd.Series.nunique, my_mode],
#         'Film_ID': [pd.Series.nunique],
#         'Film_Runtime': ['mean', 'max'],
#         'Film_Runtime': ['mean'],
        'Is_VIP': ['sum', 'mean'],
#         'Is_Outtakes': ['sum', 'mean'],
#         'Is_Licensed_Bar': ['sum', 'mean'],
#         'Is_Licensed_Lounge': ['sum', 'mean'],
        'Language': [my_mode],
        'Genre_Hierarchy_1': [my_mode],
        
        'Spend': ['mean'],
#         'Points_Earned': ['mean'],
#         'Points_Redeemed': ['mean'],
#         'Discount': ['mean'],
        
        #'Has_US_Is_LrgPopcorn': ['sum', 'mean'],
        #'Has_US_Is_Butter': ['sum', 'mean'],
        #'Has_US_Is_Alcohol': ['sum', 'mean'],
        #'Has_US_Is_Candy': ['sum', 'mean'],
        #'Has_US_Is_Kiddietray': ['sum', 'mean'],
        #'Has_US_Is_Voucher': ['sum', 'mean'],
        #'Has_US_Is_MenuFood': ['sum', 'mean'],
        #'Has_US_Is_Coffee': ['sum', 'mean'],
}).reset_index()
members_visits.columns = ["_".join(x) for x in members_visits.columns.ravel()]
members_visits = members_visits.rename(columns={'CINEPLEX_D_Membership_ID_': "CINEPLEX_D_Membership_ID"})
members_visits.columns = members_visits.columns.str.replace("<lambda_0>", "mode")
members_visits.columns = members_visits.columns.str.replace("<lambda>", "mode")

# Let's sort the members by the ones that have attended the most
# members_visits = members_visits.sort_values('Visit_Date_count', ascending=False)
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.float_format', lambda x: '%.1f' % x):
    display(members_visits.head(20))
    members_visits.shape
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.float_format', lambda x: '%.1f' % x):
    display(members_visits.describe().T)<jupyter_output><empty_output><jupyter_text>Per-Member Item Stats<jupyter_code>visits_items_big = visits_items.merge(items, on="CINEPLEX_Item_ID", how='left')
# Define my own version of the 'mode' function, which will just return the first mode (in case of tie) and thus avoid an annoying error from groupby (whcih always wants just one value)
my_mode = lambda x: pd.Series.mode(x)[0]

members_items = visits_items_big.groupby('CINEPLEX_D_Membership_ID').agg(
    {
        'CINEPLEX_Item_ID': ['count', pd.Series.nunique],
        
#         'US_Is_LrgPopcorn': ['sum', 'mean'],
#         'US_Is_Butter': ['sum', 'mean'],
#         'US_Is_Alcohol': ['sum', 'mean'],
#         'US_Is_Candy': ['sum', 'mean'],
#         'US_Is_Kiddietray': ['sum', 'mean'],
#         'US_Is_Voucher': ['sum', 'mean'],
#         'US_Is_MenuFood': ['sum', 'mean'],
#         'US_Is_Coffee': ['sum', 'mean'],
        
}).reset_index()

members_items.columns = ["_".join(x) for x in members_items.columns.ravel()]
members_items = members_items.rename(columns={'CINEPLEX_D_Membership_ID_': "CINEPLEX_D_Membership_ID"})
members_items.columns = members_items.columns.str.replace("<lambda_0>", "mode")
members_items.columns = members_items.columns.str.replace("<lambda>", "mode")

# Let's sort the members by the ones that have attended the most
members_items = members_items.sort_values('CINEPLEX_Item_ID_count', ascending=False)
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.float_format', lambda x: '%.1f' % x):
    display(members_items.head(20))
members_all = members_visits.merge(members_items, on='CINEPLEX_D_Membership_ID', how="left")
members_all = pd.get_dummies(members_all)
members_all = members_all.drop(
    ['CINEPLEX_D_Membership_ID'
    ], axis=1)
col_names = members_all.columns
# Let's write the is dataframe to disk, in case we want to use it in the future
members_all.info()
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.float_format', lambda x: '%.1f' % x):
    display(members_all.head(20))
from sklearn.metrics import silhouette_score, silhouette_samples
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering
from sklearn.mixture import GaussianMixture

scaler = StandardScaler()
members_all_scaled = pd.DataFrame(scaler.fit_transform(members_all), columns=members_all.columns)
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
pca = PCA(n_components = 9) 
PCA_components = pd.DataFrame(pca.fit_transform(members_all_scaled))

features = range(pca.n_components_)
plt.bar(features, pca.explained_variance_ratio_, color='black')
plt.xlabel('PCA features')
plt.ylabel('variance %')
plt.xticks(features);
plt.scatter(PCA_components[0], PCA_components[1], alpha=.1, color='black')
plt.xlabel('PCA 1')
plt.ylabel('PCA 2');
import matplotlib.pyplot as plt
import seaborn as sns
pca_test = PCA(n_components=13)
pca_test.fit(members_all_scaled)
sns.set(style='whitegrid')
plt.figure(figsize=(10,8))
plt.plot(np.cumsum(pca_test.explained_variance_ratio_));
plt.xlabel('number of components');
plt.ylabel('cumulative explained variance');
plt.axhline(linewidth=4, color='r', linestyle = '--', y=0.95);
display(plt.show());
evr = pca_test.explained_variance_ratio_
cvr = np.cumsum(pca_test.explained_variance_ratio_)
pca_df = pd.DataFrame()
pca_df['Cumulative Variance Ratio'] = cvr
pca_df['Explained Variance Ratio'] = evr
display(pca_df.tail(10));
from sklearn.decomposition import PCA
pca = PCA(n_components = 2) 
X_principal = pd.DataFrame(pca.fit_transform(members_all_scaled))
import scipy.cluster
aggl = scipy.cluster.hierarchy.linkage(X_principal, method='ward', metric='euclidean')
X_principal
import matplotlib.pyplot as plt

# Plot the dendogram
plt.figure(figsize=(16, 8));
plt.grid(False)
plt.title("Cineplex Dendogram");  
dend = scipy.cluster.hierarchy.dendrogram(aggl)
plt.savefig(f"Clustering/h_cluster_plot.jpg");
silhouette_scores = [] 

for n_cluster in range(2, 8):
    score = silhouette_score(X_principal, AgglomerativeClustering(n_clusters = n_cluster).fit_predict(X_principal))
    silhouette_scores.append(score)
    print(f"Clusters: {n_cluster}\nScore:", "%.4f" % score,"\n")
# Plotting a bar graph to compare the results 
k = [2, 3, 4, 5, 6,7] 
plt.figure(figsize=(12,10))
plt.bar(k, silhouette_scores) 
plt.xlabel('Number of clusters', fontsize = 10) 
plt.ylabel('Silhouette Score', fontsize = 10) 
plt.show();
from yellowbrick.cluster import SilhouetteVisualizer

fig, ax = plt.subplots(2, 2, figsize=(15,8))
for i in [2, 3, 4, 5]:
    '''
    Create KMeans instance for different number of clusters
    '''
    km = KMeans(n_clusters=i, init='k-means++', n_init=10, max_iter=100, random_state=42)
    q, mod = divmod(i, 2)
    '''
    Create SilhouetteVisualizer instance with KMeans instance
    Fit the visualizer
    '''
    visualizer = SilhouetteVisualizer(km, colors='yellowbrick', ax=ax[q-1][mod])
    visualizer.fit(X_principal)
# Create the clustering model
h_cluster = AgglomerativeClustering(n_clusters = 3)
clusters = h_cluster.fit_predict(X_principal)
# number of instances in each cluster
np.bincount(clusters)
# Add the cluster number to the original data
X_scaled_clustered = members_all
X_scaled_clustered['cluster'] = clusters
X_scaled_clustered.head()
X_scaled_clustered.groupby("cluster")['Genre_Hierarchy_1_mode_Action'].value_counts()
X_scaled_clustered.cluster.value_counts()
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.float_format', lambda x: '%.1f' % x):
    X_scaled_clustered.groupby("cluster").mean().T
X_scaled_clustered.to_csv('Clustering/clusters.csv', float_format='%.1f')
X_scaled_clustered.groupby("cluster").describe().T.to_csv('Clustering/cluster_describe.csv', float_format='%.1f')
X_scaled_clustered.groupby("cluster").mean()
X_p_clustered = X_principal
X_p_clustered = X_p_clustered.rename(columns={0:'P0', 1:'P1'})
X_p_clustered['cluster'] = clusters
X_p_clustered.head()
X_p_clustered['cluster'].value_counts()
boxplots = X_p_clustered.boxplot(by='cluster', figsize=(15,10));
boxplots;
for feature in X_scaled_clustered:
    fig = X_scaled_clustered.boxplot(column=feature, by="cluster",figsize=(14,10)).get_figure();
    fig.savefig(f"Boxplots/{feature}_plot.png");
print("Plots saved")
X_p_clustered
X_scaled_clustered.cluster.value_counts()
X_p_clustered.cluster
# Visualizing the clustering
plt.figure(figsize=(12,10))
plt.scatter(X_p_clustered['P0'], X_p_clustered['P1'],  
           c = X_p_clustered.cluster, cmap =plt.cm.winter)
plt.show();
silhouette_scores = [] 

for n_cluster in range(2, 8):
    score = silhouette_score(members_all, AgglomerativeClustering(n_clusters = n_cluster).fit_predict(members_all))
    silhouette_scores.append(score)
    print(f"Clusters: {n_cluster}\nScore:", "%.4f" % score,"\n")
# Plotting a bar graph to compare the results 
k = [2, 3, 4, 5, 6,7] 
plt.figure(figsize=(12,10))
plt.bar(k, silhouette_scores) 
plt.xlabel('Number of clusters', fontsize = 10) 
plt.ylabel('Silhouette Score', fontsize = 10) 
plt.show();
members_all_scaled_c = members_all_scaled
members_all_scaled_c['cluster'] = clusters
members_all_scaled_c['cluster'].value_counts()
def display_parallel_coordinates_centroids(df, num_clusters):
    '''Display a parallel coordinates plot for the centroids in df'''

    import seaborn as sns
    palette = sns.color_palette("bright", 10)
    # Create the plot
    fig = plt.figure(figsize=(22, 10))
    title = fig.suptitle("Parallel Coordinates plot for the Centroids", fontsize=18)
#     fig.subplots_adjust(top=0.9, bottom=0.5, wspace=0)
    plt.xticks(rotation=90)
    # Draw the chart
    ax=plt.gca()
    ax.grid(False)
    from matplotlib import rcParams
    rcParams.update({'figure.autolayout': True})
    return pd.plotting.parallel_coordinates(df, 'cluster', color=palette,axvlines=False);

    # Stagger the axes
    
    
#     for tick in ax.xaxis.get_major_ticks()[1::2]:
#         tick.set_pad(20)    
    
display_parallel_coordinates_centroids(members_all_scaled_c.groupby("cluster").mean().reset_index(), 3);
for i in range(0,68,6):
    if(i==66):
        display_parallel_coordinates_centroids(X_scaled_clustered.iloc[:, list(range(i,i+2))+[68]].groupby(by="cluster").mean().reset_index(), 3).get_figure().savefig(f"parallel_plots/parallel_plots_{i}.jpeg");  
    else:
        display_parallel_coordinates_centroids(X_scaled_clustered.iloc[:, list(range(i,i+6))+[68]].groupby(by="cluster").mean().reset_index(), 3).get_figure().savefig(f"parallel_plots/parallel_plots_{i}.jpeg");
from scipy import stats

pd.set_option("display.precision", 2)
pd.set_option('display.float_format', lambda x: '%.2f' % x)


def stats_to_df(d, scaler):
    tmp_df = pd.DataFrame(columns=col_names)
    
    tmp_df.loc[0] = scaler.inverse_transform(d.minmax[0])
    tmp_df.loc[1] = scaler.inverse_transform(d.mean)
    tmp_df.loc[2] = scaler.inverse_transform(d.minmax[1])
    tmp_df.loc[3] = scaler.inverse_transform(d.variance)
    tmp_df.loc[4] = scaler.inverse_transform(d.skewness)
    tmp_df.loc[5] = scaler.inverse_transform(d.kurtosis)
    tmp_df.index = ['Min', 'Mean', 'Max', 'Variance', 'Skewness', 'Kurtosis'] 
    return tmp_df.T

print('All Data:')
print('Number of Instances: {}'.format(members_all_scaled_c.shape[0]))
d = stats.describe(members_all_scaled_c, axis=0)
display(stats_to_df(d, scaler))

for i, label in enumerate(set(labels)):
    d = stats.describe(members_all[labels==label], axis=0)
    print('\nCluster {}:'.format(label))
    print('Number of Instances: {}'.format(d.nobs))
    display(stats_to_df(d, scaler))
# Or, we could have used panda's group-by function
import pandas as pd

X_df = pd.DataFrame(scaler.inverse_transform(members_all_scaled), columns=members_all_scaled.columns)
X_df['cluster'] = clusters

cl_group = X_df.groupby(['cluster']).agg(['count', 'mean']).T
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.float_format', lambda x: '%.1f' % x):
    display(cl_group)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14800139843841045, 'fraction_numerical': 0.017480480130520917, 'mean_word_length': 4.992667597765363, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '387399', 'license': 'Unlicense', 'n_tokens_mistral': 6445, 'n_tokens_neox': 6080, 'n_words': 1149, 'path': 'Clustering.ipynb', 'repo_name': 'Jass005/MachineLearning', 'size': '17173'}