<jupyter_start><jupyter_text>Imports<jupyter_code>import pandas as pd
from pylab import rcParams
import seaborn as sb
import matplotlib.pyplot as plt

import sklearn
from sklearn.cluster import DBSCAN
from collections import Counter
import datetime
start = datetime.datetime.now().timestamp()
# Standard data visualisaton params for Jupyter
%matplotlib inline
rcParams['figure.figsize'] = 15, 4
sb.set_style('whitegrid')<jupyter_output><empty_output><jupyter_text>Getting dataset<jupyter_code>address = '../datasets/staandelamp_realistic.json'
df_data = pd.read_json(address)

# Sort the data on timestamp
df_data = df_data.sort_values(by=['time'])



# Grab around 6 hours of timestamps
### TEMP ###
df_data = df_data[:42]
### TEMP ###

df_data.head()

# Make dates readable for debugging purposes
import datetime
import time

df_printable_dates = df_data.copy(deep=True)
df_printable_dates['dates'] = df_data['time'].map(lambda x: time.ctime(x / 1000))
df_printable_dates['printable_dates'] = df_data['time'].map(lambda x: datetime.datetime.fromtimestamp(x / 1000).isoformat())

df_printable_dates.head()<jupyter_output><empty_output><jupyter_text>reformat the first column to iterables to prevent the following error when building the DBSCAN model:ValueError: could not convert string to float: 'Staande_Lamp_5'<jupyter_code>from sklearn.preprocessing import LabelEncoder
from collections import defaultdict

print('before:\n', df_data.head())

d = defaultdict(LabelEncoder)
df_fit = df_data.apply(lambda x: d[x.name].fit_transform(x))
df_fit['time'] = df_data['time']
print('after:\n', df_fit.head())<jupyter_output>before:
              name  state           time
2  Staande_Lamp_3      0  1509489940655
6  Staande_Lamp_5      1  1509490011225
0  Staande_Lamp_1      1  1509491943009
1  Staande_Lamp_2      0  1509492221471
3  Staande_Lamp_3      1  1509492826941
after:
    name  state           time
2     2      0  1509489940655
6     4      1  1509490011225
0     0      1  1509491943009
1     1      0  1509492221471
3     2      1  1509492826941<jupyter_text>Clustering 300000ms = 5 minutes<jupyter_code>model = DBSCAN(eps=300000, min_samples=2).fit(df_fit)<jupyter_output><empty_output><jupyter_text>Visualize results<jupyter_code>df_outliers = pd.DataFrame(df_fit)
print(Counter(model.labels_))
print('amount of data:', df_fit.shape[0])
print('amount of outliers:', df_outliers[model.labels_==-1].shape[0])
print('\nsome of the outliers:\n', df_outliers[model.labels_==-1].head())<jupyter_output>Counter({-1: 12, 2: 5, 6: 4, 10: 4, 7: 3, 0: 2, 1: 2, 3: 2, 4: 2, 5: 2, 8: 2, 9: 2})
amount of data: 42
amount of outliers: 12

some of the outliers:
     name  state           time
12     3      0  1509494311740
13     4      1  1509495178373
11     2      0  1509496790877
19     3      0  1509498728417
18     2      0  1509499036484<jupyter_text>Create a color dictionary for the scatter-plot<jupyter_code>COLOR_MAP = {}
for n in set(model.labels_):
    if n % 3 == 0:
        COLOR_MAP[n] = 'firebrick'
    elif n % 3 == 1:
        COLOR_MAP[n] = 'darkgreen'
    else:
        COLOR_MAP[n] = 'darkblue'

#Outliers
COLOR_MAP[-1] = 'lightgrey'

print('All of the datapoints:\n', model.labels_)
print('\n')
print('One color for each datapoint:\n', set(model.labels_))<jupyter_output>All of the datapoints:
 [ 0  0  1  1  2  2  2  2  2 -1 -1  3  3 -1  4  4  5  5 -1 -1 -1 -1  6  6
  6  6 -1  7  7  7  8  8  9  9 -1 -1 10 10 10 10 -1 -1]


One color for each datapoint:
 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, -1}<jupyter_text>Create the graph<jupyter_code>fig = plt.figure()
ax = fig.add_axes([.1, .1, 1, 1])

colors = [COLOR_MAP[l] for l in model.labels_]

# colors = model.labels_

# use first column as X value
X = df_fit.iloc[:,2]
# set nr 1 as the value for y so it will be 1Dimentional
Y = [1] * df_fit.shape[0]

ax.scatter(X, Y, c=colors, s=20)
ax.set_xlabel = 'time'
plt.title = 'Groupingz'<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1154041687849517, 'fraction_numerical': 0.09735638027452974, 'mean_word_length': 3.888198757763975, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '608086', 'license': 'MIT', 'n_tokens_mistral': 1727, 'n_tokens_neox': 1507, 'n_words': 404, 'path': 'old/dbscan2.ipynb', 'repo_name': 'WyTho/MachineLearning', 'size': '3934'}