<jupyter_start><jupyter_text>Problem definition Cluster regions based on crime datahttp://donnees.ville.montreal.qc.ca/dataset/actes-criminels  Load the data<jupyter_code>#input
#df = pd.read_csv('data/interventionscitoyendo.csv', encoding='latin_1')
df = pd.read_csv('declarations-exterminations-punaises-de-lit-1.csv', encoding='latin_1')
#df['DATE'] = pd.to_datetime(df['DATE'])

df['DATE_DECLARATION'] = pd.to_datetime(df['DATE_DECLARATION'])
df['DATE_PRIOR_INSP'] = pd.to_datetime(df['DATE_PRIOR_INSP'])
df['DATE_FIRST_EXT'] = pd.to_datetime(df['DATE_FIRST_EXT'])
df['DATE_LAST_EXT'] = pd.to_datetime(df['DATE_LAST_EXT'])

print(df.columns)
#print(df['CATEGORIE'].value_counts())
print(df['BORO_NAME'].value_counts())
df.head()


#DATE_DECLARATION MTM8_X	MTM8_Y	LONGITUDE	LATITUDE

#del df['DATE_DECLARATION']
#del df['MTM8_X']
#del df['MTM8_Y']
#del df['LONGITUDE'] 
#del df['LATITUDE']

#del df['DEC_D'] 
#del df['PRIOR_D'] 
#del df['FIRST_D'] 
#del df['LAST_D'] 
del df['NO_DECLARATION']
del df['DATE_PRIOR_INSP']
del df['EXT_FREQ']
del df['DATE_FIRST_EXT']
del df['DATE_LAST_EXT']
del df['HOOD_NUM']
del df['HOOD_NAME'] 
#del df['BORO_NAME'] 
del df['LONG_LAT'] 
del df['MTM_X_Y']
del df['DEC_MONTH']
del df['DEC_ISSUE'] 
del df['DATE_DIFF'] 

print(df.columns)<jupyter_output>Index(['DATE_DECLARATION', 'BORO_NAME', 'MTM8_X', 'MTM8_Y', 'LONGITUDE',
       'LATITUDE'],
      dtype='object')<jupyter_text>Feature Engineering<jupyter_code># feature engineering

# select a period
#df = df[df['DATE']>='2018-01-01']
df = df[df['DATE_DECLARATION']>='2012-01-01']

# select the categories
#df = df[df['CATEGORIE']==u'Vols qualifiés']
df = df[df['BORO_NAME']==u'Verdun']

# remove lines with no location
#df = df[(df['X']>0)&(df['Y']>0)]
df = df[(df['MTM8_X']>0)&(df['MTM8_Y']>0)]


# adapt X and Y to the visualization
#df['X'] = df.apply(lambda x: transform(Proj(init='epsg:4326'), Proj(init='epsg:3857'), x['LONG'], x['LAT'])[1], axis=1)
#df['Y'] = df.apply(lambda x: transform(Proj(init='epsg:4326'), Proj(init='epsg:3857'), x['LONG'], x['LAT'])[0], axis=1)
df['MTM8_X'] = df.apply(lambda x: transform(Proj(init='epsg:4326'), Proj(init='epsg:3857'), x['LONGITUDE'], x['LATITUDE'])[1], axis=1)
df['MTM8_Y'] = df.apply(lambda x: transform(Proj(init='epsg:4326'), Proj(init='epsg:3857'), x['LONGITUDE'], x['LATITUDE'])[0], axis=1)


#X_columns = ['X', 'Y']
X_columns = ['MTM8_X', 'MTM8_Y']


df = df[X_columns]<jupyter_output><empty_output><jupyter_text>Model Training<jupyter_code>model = DBSCAN(eps=1.0, min_samples=100)
#model.fit(df[['X', 'Y']])
model.fit(df[['MTM8_X', 'MTM8_Y']])


cluster_labels = model.labels_
n_clusters = len(set(cluster_labels))
print(collections.Counter(cluster_labels))

df['cluster'] = cluster_labels
p = figure(y_range=(5641788.0, 5751788.0), x_range=(-8152883, -8252883))
p.add_tile(CARTODBPOSITRON)

#latitude  = list(df[df['cluster']>-1]['X'].values)
#longitude = list(df[df['cluster']>-1]['Y'].values)

latitude  = list(df[df['cluster']>-1]['MTM8_X'].values)
longitude = list(df[df['cluster']>-1]['MTM8_Y'].values)

colormap = list(bokeh.palettes.viridis(n_clusters))
colors = [colormap[x] for x in df[df['cluster']>-1]['cluster']]
source = ColumnDataSource(data=dict(longitude=longitude, latitude=latitude))
p.circle(x=longitude, y=latitude, color=colors, fill_alpha=0.2, size=5)
show(p)<jupyter_output><empty_output><jupyter_text>Model Evaluation<jupyter_code># Inter-Cluster
centroids = []
for cluster in sorted(set(model.labels_)):
    centroids.append(df[df['cluster']==cluster][X_columns].mean().values)
distances = []
for c1 in centroids:
    for c2 in centroids:
        distances.append(euclidean_distances(c1.reshape(-1, 1), c2.reshape(-1, 1))[0][0])
print('Inter Cluster distance', np.mean(distances))

# Intra-Cluster
distances = []
for cluster in sorted(set(model.labels_)):
    df_filter = df[df['cluster']==cluster]
    centroid = df_filter[X_columns].mean().values
    for k, v in df_filter[X_columns].iterrows():
        distances.append(euclidean_distances(centroid.reshape(-1, 1), v.values.reshape(-1, 1))[0][0])
print('Intra Cluster distance', np.mean(distances))

# Inertia
distances = []
for cluster in sorted(set(model.labels_)):
    df_filter = df[df['cluster']==cluster]
    centroid = df_filter[X_columns].mean().values
    for k, v in df_filter[X_columns].iterrows():
        distances.append(euclidean_distances(centroid.reshape(1, -1), v.values.reshape(1, -1), squared=True)[0][0])
print('Inertia', np.sum(distances))<jupyter_output>Inter Cluster distance 0.0
Intra Cluster distance 547.6135316791624
Inertia 1002676.1875
{'fraction_non_alphanumeric': 0.20082770638205183, 'fraction_numerical': 0.03768242213025485, 'mean_word_length': 7.5353159851301115, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '138263', 'license': 'MIT', 'n_tokens_mistral': 1989, 'n_tokens_neox': 1892, 'n_words': 285, 'path': 'clustering-dbscan-map-us.ipynb', 'repo_name': 'Clercy/CEBD1260_wk7', 'size': '4591'}