<jupyter_start><jupyter_text>Load Dataset<jupyter_code>item_to_id = pd.read_csv('item_to_id.csv')
item_to_id.head()
item_to_id.info()
item_to_id.nunique()
history = pd.read_csv('purchase_history.csv')
history.head()
history.info()
history.nunique()<jupyter_output><empty_output><jupyter_text>Data Processing<jupyter_code>def id_to_item(df):
    """ function to convert id into counts """
    # 'sum' here is adding two lists into one big list
    ids = df['id'].str.split(',').sum()
    id_list = [0 for i in range(1,49)]
    for i in ids:
        id_list[int(i) - 1] += 1
        
    return pd.Series(id_list, index = list(range(1,49)))
# get user_item count matrix

user_item_count = history.groupby('user_id').apply(id_to_item)

user_item_count.head()<jupyter_output><empty_output><jupyter_text>Question 1 **The customer who bought the most items overall in her lifetime**<jupyter_code>user_count = user_item_count.sum(axis = 1).reset_index().rename(columns = {0:'count'})
user_count.sort_values(by = 'count', ascending = False).head()<jupyter_output><empty_output><jupyter_text>So the customer who bought the most items is 269335 with 72 items **For each item, the customer who bought that product the most**<jupyter_code>item_user_most = user_item_count.apply(np.argmax, axis = 0).reset_index()
item_user_most = item_user_most.rename(columns = {'index': 'Item_id', 0: 'User_id'})

df = pd.merge(left = item_user_most, right = item_to_id, on = 'Item_id', how = 'left')
df[['Item_id', 'Item_name', 'User_id']]<jupyter_output><empty_output><jupyter_text>Question 2<jupyter_code># Define the feature matrix: item_id, user_id

feature = user_item_count.T
feature.head()
# determine the best number of clusters

clusters = range(2,30)
inertias = []
silhouettes = []

for n_clusters in clusters:
    kmeans = KMeans(n_clusters = n_clusters, init = 'k-means++', random_state = 42)
    kmeans = kmeans.fit(feature)
    label = kmeans.predict(feature)
    
    inertias.append(kmeans.inertia_)
    silhouettes.append(silhouette_score(feature, label))
# Visualization

fig, ax = plt.subplots(nrows = 1, ncols = 2, figsize = (18,6))
ax[0].plot(clusters, inertias, 'o-', label = 'Sum of Squared Distances')
ax[0].grid(True)
ax[1].plot(clusters, silhouettes, 'o-', label = 'Silhouette Coefficient')
ax[1].grid(True)
plt.legend(fontsize = 12)
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>It seems 12 clusters is the best choice. Now, let's apply PCA on the data and visualize the data<jupyter_code># Build K-Means model

kmeans = KMeans(n_clusters = 12, init = 'k-means++', random_state = 42)
kmeans = kmeans.fit(feature)
label = kmeans.predict(feature)
# PCA for dimension reduction

pca = PCA(n_components = 2, random_state = 42)
pca = pca.fit(feature)
pca_feature = pca.transform(feature)
# Visualization in 2D

colors = ['b','g','r','c','m','y','k','w']

fig, ax = plt.subplots(figsize = (16,16))
for j in range(len(pca_feature)):
    x = pca_feature[j][0]
    y = pca_feature[j][1]
    ax.plot(x,y,'o',color = colors[label[j] % 8])
    name = item_to_id[item_to_id['Item_id'] == j+1]['Item_name']
    ax.annotate(xy =(x+0.2, y+0.2), text=name.values[0], fontsize = 12)
    
plt.show()
# print useful information
df = pd.DataFrame({'Item_id': list(range(1, 49)), 'label': label})
df = pd.merge(left=df, right=item_to_id, on='Item_id', how='left')

for i in range(0, 21):
    print(i, '\t', list(df[df['label'] == i]['Item_name']))<jupyter_output>0 	 ['shampoo', 'shaving cream', 'hand soap', 'soap']
1 	 ['bagels', 'tortillas', 'dinner rolls', 'sandwich loaves']
2 	 ['cherries']
3 	 ['lettuce']
4 	 ['carrots', 'cauliflower']
5 	 ['sugar', 'pet items', 'baby items', 'waffles', 'sandwich bags', 'butter', 'cereals', 'eggs', 'aluminum foil', 'milk', 'laundry detergent', 'cheeses', 'frozen vegetables', 'paper towels', 'spaghetti sauce', 'dishwashing\xa0', 'canned vegetables', 'flour', 'pasta', 'toilet paper', 'ice cream', 'ketchup', 'broccoli', 'yogurt']
6 	 ['apples', 'bananas']
7 	 ['cucumbers']
8 	 ['soda', 'tea', 'juice', 'coffee']
9 	 ['poultry', 'beef', 'pork']
10 	 ['berries']
11 	 ['grapefruit']
12 	 []
13 	 []
14 	 []
15 	 []
16 	 []
17 	 []
18 	 []
19 	 []
20 	 []
{'fraction_non_alphanumeric': 0.17174119885823025, 'fraction_numerical': 0.02450047573739296, 'mean_word_length': 5.351963746223565, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '644888', 'license': 'MIT', 'n_tokens_mistral': 1718, 'n_tokens_neox': 1601, 'n_words': 366, 'path': 'Collection of Take Home Data Science Challenge/Clustering Grocery Items.ipynb', 'repo_name': 'reinfforce/Data-Science-Project', 'size': '4204'}