<jupyter_start><jupyter_text>Plotting the clusters into the folium map  clusters are showing the meeting points<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import sklearn.cluster as cluster
import time
import pandas as pd
from datetime import datetime as dt
%matplotlib inline
sns.set_context('poster')
sns.set_color_codes()
plot_kwds = {'alpha' : 0.25, 's' : 80, 'linewidths':0}

#laoding the original dataset
#converting the unix time into YearMonthDay
data=pd.read_csv('nari_dynamic.csv')
data['weeks'] = data['time'].map(lambda x: dt.fromtimestamp(x).strftime('%Y%m%d'))
data.drop(['id','time','speedoverground','courseoverground','trueheading'], axis=1, inplace=True)
data.head()
weeks=data['weeks']
weeks=weeks.unique()
weeks
#dataframe with the days reshaped to represend the weeks
weeks_df=pd.DataFrame(weeks[:182].reshape(26,7))
weeks_df
#filter the records for the chosen week
df_week1=data.copy()

df_week1 = df_week1[df_week1['weeks'].astype(int)>=int(20160121)] 
df_week1 = df_week1[['lat', 'lon','weeks']]
df_week1 = df_week1.dropna(axis=0, subset=['lat', 'lon','weeks'])

df_week1 = df_week1[df_week1['weeks'].astype(int)<=int(20160127)] 
df_week1 = df_week1[['lat', 'lon']]
df_week1 = df_week1.dropna(axis=0, subset=['lat', 'lon'])
len(df_week1)
df_week1
df_week1=df_week1.values
df_week1
#plotting the poitns of the week
plt.scatter(df_week1.T[1], df_week1.T[0], c='b', **plot_kwds)
frame = plt.gca()
frame.axes.get_xaxis()
frame.axes.get_yaxis()
#clustering algorithm by Scikit learn
def plot_clusters(data, algorithm, args, kwds):
    start_time = time.time()
    labels = algorithm(*args, **kwds).fit_predict(data)
    l=labels
    end_time = time.time()
    palette = sns.color_palette('deep', np.unique(labels).max() + 1)
    colors = [palette[x] if x >= 0 else (0.0, 0.0, 0.0) for x in labels]
    plt.scatter(data.T[1], data.T[0], c=colors, **plot_kwds)
    frame = plt.gca()
    frame.axes.get_xaxis()#.set_visible(False)
    frame.axes.get_yaxis()#.set_visible(False)
    plt.title('Clusters found by {}'.format(str(algorithm.__name__)), fontsize=24)
    plt.text(-0.5, 0.7, 'Clustering took {:.2f} s'.format(end_time - start_time), fontsize=14)
    return l
import hdbscan

d=df_week1
#HBDSCAN clustering
x=plot_clusters(d, hdbscan.HDBSCAN, (), {'min_cluster_size':50})
#extract the labels of the algorithm for each cluster
xs=pd.Series(x)
xs.value_counts()
#converting the lables into a dataframe
temp=pd.DataFrame(xs.value_counts())
temp['l']=temp.index
temp.columns = ['count', 'label']
temp
temp.describe()
temp['count'].unique()
#filtering the data
temp=temp[temp['count'] >=4000]
temp=temp[temp['label']!=-1]
temp=temp.sort_values(by=['label'])
temp
import folium 
from folium import plugins
from folium.plugins import HeatMap
from collections import Counter

fm = folium.Map(location=[48.38249, -4.4657183],
                        zoom_start=8)
df=pd.DataFrame(df_week1)
df.columns = ['lat', 'lon']
df
df['label']=x
df = df[df.label != -1]
df
#heat map for the clusters
for i in temp['label']:
    df_=df.copy()
    
    df_ = df_[df_['label']==i] # Reducing data size so it runs faster
    df_ = df_[['lat', 'lon']]
    df_ = df_.dropna(axis=0, subset=['lat','lon'])

    #points=df_.values.tolist()
    #folium.PolyLine(points, color="red", weight=2.5, opacity=1).add_to(fm)
    
    heat_data = [[row['lat'],row['lon']] for index, row in df_.iterrows()]
    HeatMap(heat_data).add_to(fm)
    
    print(i)
fm.save("week17_clus4000.html")
df_labeled_poitns=df.copy()
df_labeled_poitns.head()
#filter the dataset keeping only the lables from df_labeled_poitns
df_labeled_poitns=df_labeled_poitns.loc[df_labeled_poitns['label'].isin(temp['label'])]
#plotting the points for the map
import os

fm_meatingpoints = folium.Map(location=[48.38249, -4.4657183],
                        zoom_start=8)
c=['red', 'blue', 'green', 'purple', 'orange', 'darkred', 'lightred', 'beige', 
   'darkblue', 'darkgreen', 'cadetblue', 'darkpurple', 'white', 'pink', 'lightblue', 
   'lightgreen', 'gray', 'black', 'lightgray']
idx_c=0

#loop throught the lables and add the location to the map
for i,r in temp.iterrows():
    df_=df_labeled_poitns.copy()
    
    df_ = df_[df_['label']==r['label']] 
    df_ = df_[['lat', 'lon']]
    df_ = df_.dropna(axis=0, subset=['lat','lon'])
    print(i)
    
    #plot the points of the cluster
    for j,k in df_.iterrows():
        points=[df_.loc[j].values.tolist(),df_.loc[j].values.tolist()]
        folium.PolyLine(points, color=c[idx_c], weight=10, opacity=100).add_to(fm_meatingpoints)
        
    #clsuter information 
    #adding a marker at each cluster witht the info
    label=r['label']
    count=r['count']
    text=f'Label: {label} - Points: {count}'
    folium.Marker(location=df_.iloc[0], popup=text, icon=None,).add_to(fm_meatingpoints)
    idx_c+=1
    print('..............')

fm_meatingpoints.save("meeting_points_week17.html")<jupyter_output>147
..............
720
..............
1218
..............
1790
..............
1792
..............
1799
..............
1829
..............
{'fraction_non_alphanumeric': 0.18153303273867868, 'fraction_numerical': 0.03195451872181925, 'mean_word_length': 4.393234672304439, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '515058', 'license': 'Apache-2.0', 'n_tokens_mistral': 2004, 'n_tokens_neox': 1889, 'n_words': 407, 'path': 'meeting points.ipynb', 'repo_name': 'unic-ailab/vessel-trajectory-mining', 'size': '5101'}