<jupyter_start><jupyter_text>Global Imports<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import sklearn
import numpy as np
import time<jupyter_output><empty_output><jupyter_text>Setup Jupyter Notebook data visualisaton params<jupyter_code>from pylab import rcParams
import seaborn as sb

%matplotlib inline
rcParams['figure.figsize'] = 15, 6
sb.set_style('whitegrid')<jupyter_output><empty_output><jupyter_text>Import dataset<jupyter_code>address = '../datasets/staandelamp_realistic.json'
df_data = pd.read_json(address)

# Sort the data on timestamp
df_data = df_data.sort_values(by=['time'])

# Grab around 6 hours of timestamps
### TEMP ###
df_data = df_data[:9]   #42
### TEMP ###
df_data.iloc[0, (2)]  = 1
df_data.iloc[1, (2)]  = 2
df_data.iloc[2, (2)]  = 3
df_data.iloc[3, (2)]  = 5
df_data.iloc[4, (2)]  = 7
df_data.iloc[5, (2)]  = 9
df_data.iloc[6, (2)]  = 18
df_data.iloc[7, (2)]  = 25
df_data.iloc[8, (2)]  = 26

print(df_data.shape)

df_data.head()<jupyter_output>(9, 3)<jupyter_text>Determine the number of clusters (linkage)  only interested in column TIME<jupyter_code>X = df_data.iloc[ :, [2] ].values
#                 :        All the rows
#                    [2]   Only column 2 (time)

X[:5]<jupyter_output><empty_output><jupyter_text>Plot the timestamps and a line indicating 5 minutes<jupyter_code>from scipy.cluster.hierarchy import dendrogram, linkage
Z = linkage(X, method='complete')
gram = dendrogram(
    Z, 
    labels=X,
    # truncate_mode='lastp', 
    # p=12, 
    # leaf_rotation=45., 
    # leaf_font_size=15., 
    # show_contracted=True
)
plt.title('Dendrogram')
plt.xlabel('Timestamps')
plt.ylabel('Eucledian Distance')

# This line represents a distance of 5
plt.axhline(y=5)
# Debug Z

five_minutes = 5

print('Z[0][0]', Z[0][0])

clusters_with_distance_of_five_minutes_or_less = 0
for idx, val in enumerate(Z):
    
    print(
        '\nidx:', idx
    )
    # Count Z with a distance of less than 10 minutes
    if(val[2] < five_minutes): 
        print('< five_minutes: TRUE')
        clusters_with_distance_of_five_minutes_or_less += 1
    
    # debug a couple of the Z's
    # if(idx == 0 or idx == 1 or idx == 2 or idx == 30):
    print(
        'val:', val[0], '-', val[1], '-', val[2], '-', val[3], '\n'
    )

clusters = []
total_number_of_clusters = Z.shape[0]
print(
    clusters_with_distance_of_five_minutes_or_less, 
    'of', 
    total_number_of_clusters, 
    'clusters have a distance less than 5'
)<jupyter_output>5 of 8 clusters have a distance less than 5<jupyter_text>AgglomerativeClustering<jupyter_code>## Format the dataset to be all numbers
from sklearn.preprocessing import LabelEncoder
from collections import defaultdict

print('before:\n', df_data.head())

d = defaultdict(LabelEncoder)
df_fit = df_data.apply(lambda x: d[x.name].fit_transform(x))
df_fit['time'] = df_data['time']
print('after:\n', df_fit.head())
from sklearn.cluster import AgglomerativeClustering
import sklearn.metrics as sm

# amountOfDataRows = df_fit['name'].size

model = AgglomerativeClustering(
    affinity='euclidean', 
    compute_full_tree='auto',
    connectivity=None, 
    linkage='complete', 
    memory=None, 
    n_clusters=clusters_with_distance_of_five_minutes_or_less,
    pooling_func='deprecated'
)

model
#sm.accuracy_score(df_data.iloc[:, (2)], model.labels_)

predicted_clusters = model.fit(df_fit).labels_
# predicted_clusters = model.fit_predict(df_fit)
# predicted_clusters
### Find outliers
outlier_dict = {}
for i in model.labels_: outlier_dict[i] = i in outlier_dict
outliers = [k for k in outlier_dict if not outlier_dict[k]]

print('outliers:\n', outliers)
### Make a color map for all groups and outliers (for plotting)
COLOR_MAP = {}
group_count = 0;
for n in set(model.labels_):
    if n in outliers:
        COLOR_MAP[n] = 'lightgrey'
    else:
        if group_count % 7 == 0:
            COLOR_MAP[n] = 'firebrick'
        elif group_count % 7 == 1:
            COLOR_MAP[n] = 'darkgreen'
        elif group_count % 7 == 2:
            COLOR_MAP[n] = 'purple'
        elif group_count % 7 == 3:
            COLOR_MAP[n] = 'darkorange'
        elif group_count % 7 == 4:
            COLOR_MAP[n] = 'Teal'
        elif group_count % 7 == 5:
            COLOR_MAP[n] = 'Olive'
        elif group_count % 7 == 6:
            COLOR_MAP[n] = 'Green'
        else:
            COLOR_MAP[n] = 'darkblue'
        group_count += 1

print('All of the datapoints:\n', model.labels_)
print('One color for each datapoint:\n', set(model.labels_))
# Make figure smaller:
rcParams['figure.figsize'] = 15, 4

fig = plt.figure()
ax = fig.add_axes([.1, .1, 1, 1])

colors = [COLOR_MAP[l] for l in model.labels_]

# use first column as X value
XX = df_fit.iloc[:,2]
# set nr 1 as the value for y so it will be 1Dimentional
YY = [1] * df_fit.shape[0]

ax.scatter(XX, YY, c=colors, s=20)
ax.set_xlabel = 'time'
plt.title = 'Groupingz'<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13134992916413682, 'fraction_numerical': 0.0216555353167375, 'mean_word_length': 3.7473583093179634, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '327215', 'license': 'MIT', 'n_tokens_mistral': 1955, 'n_tokens_neox': 1821, 'n_words': 437, 'path': 'old/hca3.ipynb', 'repo_name': 'WyTho/MachineLearning', 'size': '4941'}