<jupyter_start><jupyter_text>1. Import necessary libraries<jupyter_code># Data representation and computation
import pandas as pd  
import numpy as np  
pd.options.display.float_format = '{:20,.4f}'.format

# Graph plotting
import matplotlib.pyplot as plt  
import seaborn as sns

# Data splitting, feature engg., and pipeline to train machine learning models
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import cluster, silhouette_score, v_measure_score, adjusted_rand_score, completeness_score

# Machine learning models
from sklearn.cluster import KMeans
from sklearn.cluster import DBSCAN

# Miscellaneous
import warnings
from math import log, e, ceil
from scipy.stats import entropy
from prettytable import PrettyTable

# Declaration
warnings.filterwarnings('ignore')
%precision 2
%matplotlib inline
sns.set(font_scale=1)<jupyter_output><empty_output><jupyter_text>2. Load Data<jupyter_code>ldf_insulin_data = pd.read_csv("../input/meal-data-clustering/InsulinData.csv", 
                               encoding='latin', index_col=False)

ldf_insulin_data.head()<jupyter_output><empty_output><jupyter_text>3. Data Preprocessing  3.1 Get necessary columns<jupyter_code>ldf_insulin_data = ldf_insulin_data[["Date","Time","BWZ Carb Input (grams)"]]<jupyter_output><empty_output><jupyter_text>3.2 Drop null values<jupyter_code>print("Original Data:",ldf_insulin_data.shape)

# Drop the null values
ldf_insulin_data = ldf_insulin_data.dropna()

print("After dropping null values:",ldf_insulin_data.shape)<jupyter_output>Original Data: (41435, 3)
After dropping null values: (1040, 3)<jupyter_text>3.3 Feature ScalingIt’s an important data preprocessing step for most distance-based machine learning algorithms because it can have a significant impact on the performance of algorithm.<jupyter_code>scaler = StandardScaler()
ldf_insulin_data["BWZ Carb Input (Scaled)"] = scaler.fit_transform(ldf_insulin_data['BWZ Carb Input (grams)'].values.reshape(-1, 1))<jupyter_output><empty_output><jupyter_text>4. Statistical Summary<jupyter_code>print("Number of records: "+str(len(ldf_insulin_data)))
ldf_insulin_data.describe()
ldf_insulin_data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 1040 entries, 48 to 41401
Data columns (total 4 columns):
 #   Column                   Non-Null Count  Dtype  
---  ------                   --------------  -----  
 0   Date                     1040 non-null   object 
 1   Time                     1040 non-null   object 
 2   BWZ Carb Input (grams)   1040 non-null   float64
 3   BWZ Carb Input (Scaled)  1040 non-null   float64
dtypes: float64(2), object(2)
memory usage: 40.6+ KB<jupyter_text>5. Extracting Ground Truth> 1. Derive the max and min value of meal intake amount from the Y column of the Insulin data. > 2. Discretize the meal amount in bins of size 20. > 3. According to their meal amount put them in the respective bins. > 4. Summary. In total you should have N = max-min/20  bins.  5.1 Derive the max and min value of meal intake amount from the Y - BWZ Carb Input (grams) column of the Insulin data.<jupyter_code>max_meal_amount = max(ldf_insulin_data['BWZ Carb Input (grams)'])
min_meal_amount = min(ldf_insulin_data['BWZ Carb Input (grams)'])

print("Max value of meal intake amount (grams):", max_meal_amount)
print("Min value of meal intake amount (grams):", min_meal_amount)
number_of_bins = ceil((max_meal_amount-min_meal_amount)/20)
print("In total you should have N = (",max_meal_amount,"-",min_meal_amount,"/ 20) i.e.", number_of_bins, "bins")<jupyter_output>In total you should have N = ( 129.0 - 0.0 / 20) i.e. 7 bins<jupyter_text>5.2 Discretize the meal amount in bins of size 20.<jupyter_code># Define bins
ldict_bins = {
    1:[0,20],
    2:[21,40],
    3:[41,60],
    4:[61,80],
    5:[81,100],
    6:[101,120],
    7:[121,140]
}
def get_bin(meal_amount_in_grams):
    """
    This method is used to get appropriate bin bucket 
    using the bin dictionary defined above.
    """
    lint_bin = 0
    for bin_number, bin_range in ldict_bins.items():
        if bin_range[0]<=meal_amount_in_grams<=bin_range[1]:
            lint_bin = bin_number
    return lint_bin<jupyter_output><empty_output><jupyter_text>5.3 According to their meal amount put them in the respective bins.<jupyter_code>ldf_insulin_data['Ground Truth'] = ldf_insulin_data['BWZ Carb Input (grams)'].apply(lambda x: get_bin(x))<jupyter_output><empty_output><jupyter_text>5.4 Summary<jupyter_code>ldf_insulin_data["Ground Truth"].value_counts().sort_values(ascending=True).plot(kind="barh")
ldf_insulin_data.head()
ldf_insulin_data['Ground Truth'].values[:5]<jupyter_output><empty_output><jupyter_text>6. Perform clustering> 1. Feature Selection. > 2. Methods to calculate accuracy based on SSE, entropy and purity metrics> 2. KMeans Clustering. > 3. DBSCAN Clustering. > 4. Accuracy report  6.1 Feature Selection<jupyter_code>X = ldf_insulin_data["BWZ Carb Input (Scaled)"].values.reshape(-1,1)<jupyter_output><empty_output><jupyter_text>6.2 Methods to calculate accuracy based on SSE, entropy and purity metrics<jupyter_code>def calculate_entropy(y_true, y_pred, base = 2):
    """
    """
    contingency_matrix = cluster.contingency_matrix(y_true, y_pred)
    base = e if base is None else base
    
    Entropy = []

    for i in range(0, len(contingency_matrix)):
        p = contingency_matrix[i,:]
        p = pd.Series(p).value_counts(normalize=True, sort=False)
        Entropy.append((-p/p.sum() * np.log(p/p.sum())/np.log(2)).sum())
    
    TotalP = sum(contingency_matrix,1);
    WholeEntropy = 0;

    for i in range(0, len(contingency_matrix)):
        p = contingency_matrix[i,:]
        WholeEntropy = WholeEntropy + ((sum(p))/(sum(TotalP)))*Entropy[i]
    
    return WholeEntropy
def calculate_purity_score(y_true, y_pred):
    """
    Purity is an external evaluation criterion of cluster quality. 
    It is the percent of the total number of objects(data points) that were classified correctly.
    """
    # compute contingency matrix (also called confusion matrix)
    contingency_matrix = cluster.contingency_matrix(y_true, y_pred)

    Purity = []

    for i in range(0, len(contingency_matrix)):
        p = contingency_matrix[i,:]
        Purity.append(p.max()/p.sum())

    TotalP = sum(contingency_matrix,1);
    WholePurity = 0;

    for i in range(0, len(contingency_matrix)):
        p = contingency_matrix[i,:]
        WholePurity = WholePurity + ((sum(p))/(sum(TotalP)))*Purity[i]
    
    return WholePurity
def calculate_v_measure_score(y_true, y_pred):
    """
    This metric is independent of the absolute values of the labels: 
    a permutation of the class or cluster label values won’t change the score value in any way.
    """
    return v_measure_score(y_true, y_pred)<jupyter_output><empty_output><jupyter_text>6.3 KMeans ClusteringK-Means cluster is one of the most commonly used unsupervised machine learning clustering techniques. It is a centroid based clustering technique that needs you decide the number of clusters (centroids) and randomly places the cluster centroids to begin the clustering process. The goal is to divide N observations into K clusters repeatedly until no more groups can be formed. Advantages of K-Means1. Easy to understand and implement.2. Can handle large datasets well. Disadvantages of K-Means1. Sensitive to number of clusters/centroids chosen. Even after using techniques like Elbow method, it is sometimes hard to generate good clusters.2. Does not work well with outliers. Centroids can get dragged by the outliers resulting in skewed clusters.3. Gets difficult in high dimensional spaces as the distance between the points increases and Euclidean distance diverges (converges to a constant value).4. Gets slow as the number of dimensions increases.<jupyter_code># Define a model
kmeans = KMeans(n_clusters=7, random_state=42, max_iter=100)

# Fit a model
kmeans_model = kmeans.fit(X)
# The SSE value
print("The SSE value:\n"+ str(kmeans.inertia_))

# The number of iterations required to converge
print("\nThe number of iterations required to converge:\n"+str(kmeans.n_iter_))

# Compute the silhouette scores
kmeans_silhouette = silhouette_score(
    X, kmeans.labels_
).round(2)

print("\nSilhouette Score (A higher silhouette coefficient suggests better clusters):\n"+str(kmeans_silhouette))
kmeans.labels_[:5]<jupyter_output><empty_output><jupyter_text>Note that the order of the cluster labels are flipped. The order was [2, 1, 1, 4, 1] in true_labels but [2, 1, 6, 5, 6] in kmeans.labels_ even though those data objects are still members of their original clusters in kmeans.lables_.This behavior is normal, as the ordering of cluster labels is dependent on the initialization. Cluster 0 from the first run could be labeled cluster 1 in the second run and vice versa. This doesn’t affect clustering evaluation metrics.<jupyter_code># Create a column of K-means Prediction
ldf_insulin_data['KmeanCluster'] = kmeans_model.predict(X)
# Calculate accuracy using entropy, purity_score, & v-measure score
kmean_entropy = calculate_entropy(ldf_insulin_data['Ground Truth'],ldf_insulin_data['KmeanCluster'])
kmean_purity_score = calculate_purity_score(ldf_insulin_data['Ground Truth'],ldf_insulin_data['KmeanCluster'])
kmean_v_measure_score = calculate_v_measure_score(ldf_insulin_data['Ground Truth'], ldf_insulin_data['KmeanCluster'])<jupyter_output><empty_output><jupyter_text>6.4 DBSCAN ClusteringDBSCAN is a density-based clustering algorithm that forms clusters of dense regions of data points ignoring the low-density areas (considering them as noise). Advantages of DBSCAN* Works well for noisy datasets.* Can identity Outliers easily.* Clusters can take any irregular shape unlike K-Means where clusters are more or less spherical. Disadvantages of DBSCAN* Does not work very well for sparse datasets or datasets with varying density.* Sensitive to eps and minPts parameters.* Not partitionable for multiprocessor systems.<jupyter_code># Define a model
dbscan = DBSCAN(eps=0.3)

# Fit a model
dbscan_model = dbscan.fit(X)
# Compute the silhouette scores
dbscan_silhouette = silhouette_score(
    X, dbscan_model.labels_
).round(2)

print("Silhouette Score (A higher silhouette coefficient suggests better clusters):\n"+str(dbscan_silhouette))
# Create a column of DBSCAN Prediction
ldf_insulin_data['DBSCAN_Cluster'] = dbscan_model.fit_predict(X)
# Calculate accuracy using entropy, purity_score, & v-measure score
dbscan_entropy = calculate_entropy(ldf_insulin_data['Ground Truth'],ldf_insulin_data['DBSCAN_Cluster'])
dbscan_purity_score = calculate_purity_score(ldf_insulin_data['Ground Truth'],ldf_insulin_data['DBSCAN_Cluster'])
dbscan_v_measure_score = calculate_v_measure_score(ldf_insulin_data['Ground Truth'], ldf_insulin_data['DBSCAN_Cluster'])<jupyter_output><empty_output><jupyter_text>6.5 Accuracy report<jupyter_code>Model_Accuracy = PrettyTable()
Model_Accuracy.field_names = ["","SSE","V-Measure Score", "Entropy", "Purity Metrics"]
Model_Accuracy.align[""] = "r"
Model_Accuracy.add_row(["K-means (K=7)","%.2f"%kmeans.inertia_,"%.2f"%kmean_v_measure_score,"%.2f"%kmean_entropy,"%.2f"%kmean_purity_score])
Model_Accuracy.add_row(["DBSCAN","-","%.2f"%dbscan_v_measure_score,"%.2f"%dbscan_entropy,"%.2f"%dbscan_purity_score])
print("Accuracy report\n")
print(Model_Accuracy)<jupyter_output>Accuracy report

+---------------+-------+-----------------+---------+----------------+
|               |  SSE  | V-Measure Score | Entropy | Purity Metrics |
+---------------+-------+-----------------+---------+----------------+
| K-means (K=7) | 22.11 |       0.65      |   1.39  |      0.61      |
|        DBSCAN |   -   |       0.01      |   1.00  |      1.00      |
+---------------+-------+-----------------+---------+----------------+<jupyter_text>Export Result.csv<jupyter_code>ldct_result = {}

ldct_result['SSE for Kmeans'] =  "%.2f"%kmeans.inertia_
ldct_result['SSE for DBSCAN'] =  "-"
ldct_result['Entropy for Kmeans'] =  "%.2f"%kmean_entropy
ldct_result['Entropy for DBSCAN'] =  "%.2f"%dbscan_entropy
ldct_result['Purity for K means'] =  "%.2f"%kmean_purity_score
ldct_result['Purity for DBSCAN'] =  "%.2f"%dbscan_purity_score

ldf_result = pd.DataFrame(ldct_result, index=[0])
ldf_result.to_csv('Result.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11906690426048923, 'fraction_numerical': 0.01960149036125061, 'mean_word_length': 4.574266365688487, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '438234', 'license': 'MIT', 'n_tokens_mistral': 4252, 'n_tokens_neox': 3929, 'n_words': 1093, 'path': 'Meal Data Clustering.ipynb', 'repo_name': 'rahulgaikwad2010/meal-data-clustering', 'size': '12346'}